/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.IInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.*;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
import org.testng.SkipException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.*;
import static org.testng.Assert.assertEquals;
/**
* Test utility class.
*/
public final class TestUtils {
public static final long TEST_DATE_IN_LONG = 1418265358440L;
public static final String EMPLOYEES_ATTR = "employees";
public static final String DEPARTMENT_ATTR = "department";
public static final String ASSETS_ATTR = "assets";
public static final String POSITIONS_ATTR = "positions";
public static final String ASSET_TYPE = "TestAsset";
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
public static final String PROCESS_TYPE = "hive_process";
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
public static final String SUPER_TYPE_NAME = "Base";
public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
public static final String PARTITION_CLASS_TYPE = "partition_class_type";
public static final String SERDE_TYPE = "serdeType";
public static final String COLUMNS_MAP = "columnsMap";
public static final String COLUMNS_ATTR_NAME = "columns";
public static final String NAME = "name";
private TestUtils() {
}
/**
* Dumps the graph in GSON format in the path returned.
*
* @param graph handle to graph
* @return path to the dump file
* @throws Exception
*/
public static String dumpGraph(AtlasGraph<?,?> graph) throws Exception {
File tempFile = File.createTempFile("graph", ".gson");
System.out.println("tempFile.getPath() = " + tempFile.getPath());
GraphHelper.dumpToLog(graph);
FileOutputStream os = null;
try {
os = new FileOutputStream(tempFile);
graph.exportToGson(os);
}
finally {
if(os != null) {
try {
os.close();
}
catch(IOException e) {
e.printStackTrace();
}
}
}
return tempFile.getPath();
}
/**
* Class Hierarchy is:
* Department(name : String, employees : Array[Person])
* Person(name : String, department : Department, manager : Manager)
* Manager(subordinates : Array[Person]) extends Person
* <p/>
* Persons can have SecurityClearance(level : Int) clearance.
*/
public static void defineDeptEmployeeTypes(TypeSystem ts) throws AtlasException {
String _description = "_description";
EnumTypeDefinition orgLevelEnum =
new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1), new EnumValue("L2", 2));
StructTypeDefinition addressDetails =
createStructTypeDef("Address", "Address"+_description, createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createRequiredAttrDef("city", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef(DEPARTMENT_TYPE, "Department"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
new AttributeDefinition(EMPLOYEES_ATTR, String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
true, DEPARTMENT_ATTR),
new AttributeDefinition(POSITIONS_ATTR, String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL,
false, null)
);
HierarchicalTypeDefinition<ClassType> personTypeDef = createClassTypeDef("Person", "Person"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createOptionalAttrDef("orgLevel", "OrgLevel"),
createOptionalAttrDef("address", "Address"),
new AttributeDefinition(DEPARTMENT_ATTR, "Department", Multiplicity.REQUIRED, false, EMPLOYEES_ATTR),
new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"),
new AttributeDefinition("mentor", "Person", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition(ASSETS_ATTR, String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null),
createOptionalAttrDef("birthday", DataTypes.DATE_TYPE),
createOptionalAttrDef("hasPets", DataTypes.BOOLEAN_TYPE),
createOptionalAttrDef("numberOfCars", DataTypes.BYTE_TYPE),
createOptionalAttrDef("houseNumber", DataTypes.SHORT_TYPE),
createOptionalAttrDef("carMileage", DataTypes.INT_TYPE),
createOptionalAttrDef("shares", DataTypes.LONG_TYPE),
createOptionalAttrDef("salary", DataTypes.DOUBLE_TYPE),
createOptionalAttrDef("age", DataTypes.FLOAT_TYPE),
createOptionalAttrDef("numberOfStarsEstimate", DataTypes.BIGINTEGER_TYPE),
createOptionalAttrDef("approximationOfPi", DataTypes.BIGDECIMAL_TYPE),
createOptionalAttrDef("isOrganDonor", DataTypes.BOOLEAN_TYPE)
);
HierarchicalTypeDefinition<ClassType> assetTypeDef = createClassTypeDef(ASSET_TYPE, "Asset"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
new AttributeDefinition("childAssets", String.format("array<%s>", ASSET_TYPE) , Multiplicity.OPTIONAL, false, null)
);
HierarchicalTypeDefinition<ClassType> managerTypeDef = createClassTypeDef("Manager", "Manager"+_description, ImmutableSet.of("Person"),
new AttributeDefinition("subordinates", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
false, "manager"));
HierarchicalTypeDefinition<TraitType> securityClearanceTypeDef =
createTraitTypeDef("SecurityClearance", "SecurityClearance"+_description, ImmutableSet.<String>of(),
createRequiredAttrDef("level", DataTypes.INT_TYPE));
ts.defineTypes(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
ImmutableList.of(securityClearanceTypeDef),
ImmutableList.of(deptTypeDef, personTypeDef, managerTypeDef, assetTypeDef));
}
public static final String DEPARTMENT_TYPE = "Department";
public static final String PERSON_TYPE = "Person";
public static ITypedReferenceableInstance createDeptEg1(TypeSystem ts) throws AtlasException {
Referenceable hrDept = new Referenceable(DEPARTMENT_TYPE);
Referenceable john = new Referenceable(PERSON_TYPE);
Referenceable jane = new Referenceable("Manager", "SecurityClearance");
Referenceable johnAddr = new Referenceable("Address");
Referenceable janeAddr = new Referenceable("Address");
Referenceable julius = new Referenceable("Manager");
Referenceable juliusAddr = new Referenceable("Address");
Referenceable max = new Referenceable("Person");
Referenceable maxAddr = new Referenceable("Address");
hrDept.set(NAME, "hr");
john.set(NAME, "John");
john.set(DEPARTMENT_ATTR, hrDept);
johnAddr.set("street", "Stewart Drive");
johnAddr.set("city", "Sunnyvale");
john.set("address", johnAddr);
john.set("birthday",new Date(1950, 5, 15));
john.set("isOrganDonor", true);
john.set("hasPets", true);
john.set("numberOfCars", 1);
john.set("houseNumber", 153);
john.set("carMileage", 13364);
john.set("shares", 15000);
john.set("salary", 123345.678);
john.set("age", 50);
john.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000"));
john.set("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286"));
jane.set(NAME, "Jane");
jane.set(DEPARTMENT_ATTR, hrDept);
janeAddr.set("street", "Great America Parkway");
janeAddr.set("city", "Santa Clara");
jane.set("address", janeAddr);
janeAddr.set("street", "Great America Parkway");
julius.set(NAME, "Julius");
julius.set(DEPARTMENT_ATTR, hrDept);
juliusAddr.set("street", "Madison Ave");
juliusAddr.set("city", "Newtonville");
julius.set("address", juliusAddr);
julius.set("subordinates", ImmutableList.<Referenceable>of());
max.set(NAME, "Max");
max.set(DEPARTMENT_ATTR, hrDept);
maxAddr.set("street", "Ripley St");
maxAddr.set("city", "Newton");
max.set("address", maxAddr);
max.set("manager", jane);
max.set("mentor", julius);
max.set("birthday",new Date(1979, 3, 15));
max.set("isOrganDonor", true);
max.set("hasPets", true);
max.set("age", 36);
max.set("numberOfCars", 2);
max.set("houseNumber", 17);
max.set("carMileage", 13);
max.set("shares", Long.MAX_VALUE);
max.set("salary", Double.MAX_VALUE);
max.set("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000"));
max.set("approximationOfPi", new BigDecimal("3.1415926535897932"));
john.set("manager", jane);
john.set("mentor", max);
hrDept.set(EMPLOYEES_ATTR, ImmutableList.of(john, jane, julius, max));
jane.set("subordinates", ImmutableList.of(john, max));
jane.getTrait("SecurityClearance").set("level", 1);
ClassType deptType = ts.getDataType(ClassType.class, "Department");
ITypedReferenceableInstance hrDept2 = deptType.convert(hrDept, Multiplicity.REQUIRED);
Assert.assertNotNull(hrDept2);
return hrDept2;
}
public static TypesDef simpleType(){
HierarchicalTypeDefinition<ClassType> superTypeDefinition =
createClassTypeDef("h_type", ImmutableSet.<String>of(),
createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)});
HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
EnumValue values[] = {new EnumValue("ONE", 1),};
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition));
}
public static TypesDef simpleTypeUpdated(){
HierarchicalTypeDefinition<ClassType> superTypeDefinition =
createClassTypeDef("h_type", ImmutableSet.<String>of(),
createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE)});
HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
EnumValue values[] = {new EnumValue("ONE", 1),};
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition));
}
public static TypesDef simpleTypeUpdatedDiff() {
HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition));
}
public static TypesDef defineHiveTypes() {
String _description = "_description";
HierarchicalTypeDefinition<ClassType> superTypeDefinition =
createClassTypeDef(SUPER_TYPE_NAME, ImmutableSet.<String>of(),
createOptionalAttrDef("namespace", DataTypes.STRING_TYPE),
createOptionalAttrDef("cluster", DataTypes.STRING_TYPE),
createOptionalAttrDef("colo", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
createClassTypeDef(DATABASE_TYPE, DATABASE_TYPE + _description,ImmutableSet.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createOptionalAttrDef("created", DataTypes.DATE_TYPE),
createOptionalAttrDef("isReplicated", DataTypes.BOOLEAN_TYPE),
new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),
createRequiredAttrDef("description", DataTypes.STRING_TYPE));
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", "serdeType" + _description,
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", "tableType" + _description, values);
HierarchicalTypeDefinition<ClassType> columnsDefinition =
createClassTypeDef(COLUMN_TYPE, ImmutableSet.<String>of(),
createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE));
StructTypeDefinition partitionDefinition = new StructTypeDefinition("partition_struct_type", "partition_struct_type" + _description,
new AttributeDefinition[]{createRequiredAttrDef(NAME, DataTypes.STRING_TYPE),});
AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
new AttributeDefinition("location", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false,
null),
new AttributeDefinition("numBuckets", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
};
HierarchicalTypeDefinition<ClassType> storageDescClsDef =
new HierarchicalTypeDefinition<>(ClassType.class, STORAGE_DESC_TYPE, STORAGE_DESC_TYPE + _description,
ImmutableSet.of(SUPER_TYPE_NAME), attributeDefinitions);
AttributeDefinition[] partClsAttributes = new AttributeDefinition[]{
new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("table", TABLE_TYPE, Multiplicity.REQUIRED, false, null),
new AttributeDefinition("createTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("lastAccessTime", DataTypes.LONG_TYPE.getName(), Multiplicity.OPTIONAL, false,
null),
new AttributeDefinition("sd", STORAGE_DESC_TYPE, Multiplicity.REQUIRED, true,
null),
new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null),
new AttributeDefinition("parameters", new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), Multiplicity.OPTIONAL, false, null),};
HierarchicalTypeDefinition<ClassType> partClsDef =
new HierarchicalTypeDefinition<>(ClassType.class, "partition_class_type", "partition_class_type" + _description,
ImmutableSet.of(SUPER_TYPE_NAME), partClsAttributes);
HierarchicalTypeDefinition<ClassType> processClsType =
new HierarchicalTypeDefinition<>(ClassType.class, PROCESS_TYPE, PROCESS_TYPE + _description,
ImmutableSet.<String>of(), new AttributeDefinition[]{
new AttributeDefinition("outputs", "array<" + TABLE_TYPE + ">", Multiplicity.OPTIONAL, false, null)
});
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
createClassTypeDef(TABLE_TYPE, TABLE_TYPE + _description, ImmutableSet.of(SUPER_TYPE_NAME),
TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE),
createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createRequiredAttrDef("type", DataTypes.STRING_TYPE),
createOptionalAttrDef("created", DataTypes.DATE_TYPE),
// enum
new AttributeDefinition("tableType", "tableType", Multiplicity.REQUIRED, false, null),
// array of strings
new AttributeDefinition("columnNames",
String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL,
false, null),
// array of classes
new AttributeDefinition("columns", String.format("array<%s>", COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null),
// array of structs
new AttributeDefinition("partitions", String.format("array<%s>", "partition_struct_type"),
Multiplicity.OPTIONAL, true, null),
// map of primitives
new AttributeDefinition("parametersMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null),
//map of classes -
new AttributeDefinition(COLUMNS_MAP,
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
COLUMN_TYPE),
Multiplicity.OPTIONAL, true, null),
//map of structs
new AttributeDefinition("partitionsMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"partition_struct_type"),
Multiplicity.OPTIONAL, true, null),
// struct reference
new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null),
// class reference
new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
//class reference as composite
new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
createTraitTypeDef(PII, PII + _description, ImmutableSet.<String>of());
HierarchicalTypeDefinition<TraitType> classificationTypeDefinition =
createTraitTypeDef(CLASSIFICATION, CLASSIFICATION + _description, ImmutableSet.<String>of(),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<TraitType> fetlClassificationTypeDefinition =
createTraitTypeDef("fetl" + CLASSIFICATION, "fetl" + CLASSIFICATION + _description, ImmutableSet.of(CLASSIFICATION),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition),
ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition,
storageDescClsDef, partClsDef, processClsType));
}
public static Collection<IDataType> createHiveTypes(TypeSystem typeSystem) throws Exception {
if (!typeSystem.isRegistered(TABLE_TYPE)) {
TypesDef typesDef = defineHiveTypes();
return typeSystem.defineTypes(typesDef).values();
}
return new ArrayList<>();
}
public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10);
}
public static Referenceable createDBEntity() {
Referenceable entity = new Referenceable(DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10);
entity.set(NAME, dbName);
entity.set("description", "us db");
return entity;
}
public static Referenceable createTableEntity(String dbId) {
Referenceable entity = new Referenceable(TABLE_TYPE);
String tableName = RandomStringUtils.randomAlphanumeric(10);
entity.set(NAME, tableName);
entity.set("description", "random table");
entity.set("type", "type");
entity.set("tableType", "MANAGED");
entity.set("database", new Id(dbId, 0, DATABASE_TYPE));
entity.set("created", new Date());
return entity;
}
public static Referenceable createColumnEntity() {
Referenceable entity = new Referenceable(COLUMN_TYPE);
entity.set(NAME, RandomStringUtils.randomAlphanumeric(10));
entity.set("type", "VARCHAR(32)");
return entity;
}
/**
* Creates an entity in the graph and does basic validation
* of the GuidMapping that was created in the process.
*
*/
public static String createInstance(MetadataService metadataService, Referenceable entity) throws Exception {
RequestContext.createContext();
String entityjson = InstanceSerialization.toJson(entity, true);
JSONArray entitiesJson = new JSONArray();
entitiesJson.put(entityjson);
CreateUpdateEntitiesResult creationResult = metadataService.createEntities(entitiesJson.toString());
Map<String,String> guidMap = creationResult.getGuidMapping().getGuidAssignments();
Map<Id, Referenceable> referencedObjects = findReferencedObjects(entity);
for(Map.Entry<Id,Referenceable> entry : referencedObjects.entrySet()) {
Id foundId = entry.getKey();
if(foundId.isUnassigned()) {
String guid = guidMap.get(entry.getKey()._getId());
Referenceable obj = entry.getValue();
loadAndDoSimpleValidation(guid,obj, metadataService);
}
}
List<String> guids = creationResult.getCreatedEntities();
if (guids != null && guids.size() > 0) {
return guids.get(guids.size() - 1);
}
return null;
}
private static Map<Id,Referenceable> findReferencedObjects(Referenceable ref) {
Map<Id, Referenceable> result = new HashMap<>();
findReferencedObjects(ref, result);
return result;
}
private static void findReferencedObjects(Referenceable ref, Map<Id, Referenceable> seen) {
Id guid = ref.getId();
if(seen.containsKey(guid)) {
return;
}
seen.put(guid, ref);
for(Map.Entry<String, Object> attr : ref.getValuesMap().entrySet()) {
Object value = attr.getValue();
if(value instanceof Referenceable) {
findReferencedObjects((Referenceable)value, seen);
}
else if(value instanceof List) {
for(Object o : (List)value) {
if(o instanceof Referenceable) {
findReferencedObjects((Referenceable)o, seen);
}
}
}
else if(value instanceof Map) {
for(Object o : ((Map)value).values()) {
if(o instanceof Referenceable) {
findReferencedObjects((Referenceable)o, seen);
}
}
}
}
}
/**
* Clears the state in the request context.
*
*/
public static void resetRequestContext() {
//reset the context while preserving the user
String user = RequestContext.get().getUser();
RequestContext.createContext();
RequestContext.get().setUser(user);
}
/**
* Triggers the Atlas initialization process using the specified MetadataRepository.
* This causes the built-in types and their indices to be created.
*/
public static void setupGraphProvider(MetadataRepository repo) throws AtlasException {
TypeCache typeCache = null;
try {
typeCache = AtlasRepositoryConfiguration.getTypeCache().newInstance();
}
catch(Throwable t) {
typeCache = new DefaultTypeCache();
}
final GraphBackedSearchIndexer indexer = new GraphBackedSearchIndexer(new AtlasTypeRegistry());
Provider<TypesChangeListener> indexerProvider = new Provider<TypesChangeListener>() {
@Override
public TypesChangeListener get() {
return indexer;
}
};
Configuration config = ApplicationProperties.get();
ITypeStore typeStore = new GraphBackedTypeStore();
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo,
typeStore,
Collections.singletonList(indexerProvider),
new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache);
//commit the created types
getGraph().commit();
}
public static AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
}
/**
* Adds a proxy wrapper around the specified MetadataService that automatically
* resets the request context before every call.
*
* @param delegate
* @return
*/
public static MetadataService addSessionCleanupWrapper(final MetadataService delegate) {
return (MetadataService)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
new Class[]{MetadataService.class}, new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
try {
resetRequestContext();
Object result = method.invoke(delegate, args);
return result;
}
catch(InvocationTargetException e) {
e.getCause().printStackTrace();
throw e.getCause();
}
catch(Throwable t) {
t.printStackTrace();
throw t;
}
}
});
}
/**
* Adds a proxy wrapper around the specified MetadataRepository that automatically
* resets the request context before every call and either commits or rolls
* back the graph transaction after every call.
*
* @param delegate
* @return
*/
public static MetadataRepository addTransactionWrapper(final MetadataRepository delegate) {
return (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
new Class[]{MetadataRepository.class}, new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
boolean useTransaction = GraphBackedMetadataRepository.class.getMethod(
method.getName(), method.getParameterTypes())
.isAnnotationPresent(GraphTransaction.class);
try {
resetRequestContext();
Object result = method.invoke(delegate, args);
if(useTransaction) {
System.out.println("Committing changes");
getGraph().commit();
System.out.println("Commit succeeded.");
}
return result;
}
catch(InvocationTargetException e) {
e.getCause().printStackTrace();
if(useTransaction) {
System.out.println("Rolling back changes due to exception.");
getGraph().rollback();
}
throw e.getCause();
}
catch(Throwable t) {
t.printStackTrace();
if(useTransaction) {
System.out.println("Rolling back changes due to exception.");
getGraph().rollback();
}
throw t;
}
}
});
}
/**
* Loads the entity and does sanity testing of the GuidMapping that was
* created during the operation.
*
*/
public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataRepository repositoryService) throws AtlasException {
ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid);
doSimpleValidation(original, loaded);
return loaded;
}
/**
* Loads the entity and does sanity testing of the GuidMapping that was
* created during the operation.
*
*/
public static ITypedReferenceableInstance loadAndDoSimpleValidation(String guid, Referenceable original, MetadataService repositoryService) throws AtlasException {
ITypedReferenceableInstance loaded = repositoryService.getEntityDefinition(guid);
doSimpleValidation(original, loaded);
return loaded;
}
private static void doSimpleValidation(Referenceable original, IInstance loaded) throws AtlasException {
assertEquals(loaded.getTypeName(), original.getTypeName());
ClassType ct = TypeSystem.getInstance().getDataType(ClassType.class, loaded.getTypeName());
//compare primitive fields
for(AttributeInfo field : ct.fieldMapping.fields.values()) {
if(field.dataType().getTypeCategory() == TypeCategory.PRIMITIVE) {
if(original.get(field.name) != null) {
Object rawLoadedValue = loaded.get(field.name);
Object rawProvidedValue = original.get(field.name);
Object convertedLoadedValue = field.dataType().convert(rawLoadedValue, Multiplicity.REQUIRED);
Object convertedProvidedValue = field.dataType().convert(rawProvidedValue, Multiplicity.REQUIRED);
assertEquals(convertedLoadedValue, convertedProvidedValue);
}
}
}
}
/**
* Validates that the two String Collections contain the same items, without
* regard to order.
*
*/
public static void assertContentsSame(Collection<String> actual, Collection<String> expected) {
assertEquals(actual.size(), expected.size());
Set<String> checker = new HashSet<>();
checker.addAll(expected);
checker.removeAll(actual);
assertEquals(checker.size(), 0);
}
public static void skipForGremlin3EnabledGraphDb() throws SkipException {
//ATLAS-1579 Currently, some tests are skipped for titan1 backened. As these tests are hard coded to use Gremlin2. See ATLAS-1579, ATLAS-1591 once it is fixed, please remove it.
if (TestUtils.getGraph().getSupportedGremlinVersion() == GremlinVersion.THREE) {
throw new SkipException ("This test requires Gremlin2. Skipping test ");
}
}
}