GraphBackedTypeStoreTest.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.atlas.repository.typestore;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.atlas.AtlasException;
import org.apache.atlas.TestOnlyModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.*;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;

import javax.inject.Inject;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;

@Guice(modules = TestOnlyModule.class)
public class GraphBackedTypeStoreTest {
    
    private static final String DESCRIPTION = "_description";

    @Inject
    private ITypeStore typeStore;

    private TypeSystem ts;

    @BeforeClass
    public void setUp() throws Exception {
        ts = TypeSystem.getInstance();
        ts.reset();
        TestUtils.defineDeptEmployeeTypes(ts);
    }

    @AfterClass
    public void tearDown() throws Exception {
        ts.reset();
        AtlasGraphProvider.cleanup();
    }


    @Test
    public void testStore() throws AtlasException {
        ImmutableList<String> typeNames = ts.getTypeNames();
        typeStore.store(ts, typeNames);
        dumpGraph();
    }

    @Test(dependsOnMethods = "testStore")
    public void testRestoreType() throws Exception {
        TypesDef typesDef = typeStore.restoreType("Manager");
        verifyRestoredClassType(typesDef, "Manager");
    }

    private void dumpGraph() {
        AtlasGraph<?, ?> graph = TestUtils.getGraph();
        for (AtlasVertex<?,?> v : graph.getVertices()) {
            System.out.println("****v = " + GraphHelper.vertexString(v));
            for (AtlasEdge<?,?> e : v.getEdges(AtlasEdgeDirection.OUT)) {
                System.out.println("****e = " + GraphHelper.edgeString(e));
            }
        }
    }

    @Test(dependsOnMethods = "testStore")
    public void testRestore() throws Exception {
        TypesDef types = typeStore.restore();

        //validate enum
        List<EnumTypeDefinition> enumTypes = types.enumTypesAsJavaList();
        Assert.assertEquals(1, enumTypes.size());
        EnumTypeDefinition orgLevel = enumTypes.get(0);
        Assert.assertEquals(orgLevel.name, "OrgLevel");
        Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION);
        Assert.assertEquals(orgLevel.enumValues.length, 2);
        EnumValue enumValue = orgLevel.enumValues[0];
        Assert.assertEquals(enumValue.value, "L1");
        Assert.assertEquals(enumValue.ordinal, 1);

        //validate class
        List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
        Assert.assertEquals(1, structTypes.size());

        verifyRestoredClassType(types, "Manager");

        //validate trait
        List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
        Assert.assertEquals(1, traitTypes.size());
        HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
        Assert.assertEquals("SecurityClearance", trait.typeName);
        Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription);
        Assert.assertEquals(1, trait.attributeDefinitions.length);
        AttributeDefinition attribute = trait.attributeDefinitions[0];
        Assert.assertEquals("level", attribute.name);
        Assert.assertEquals(DataTypes.INT_TYPE.getName(), attribute.dataTypeName);

        //validate the new types
        ts.reset();
        ts.defineTypes(types);
    }

    @Test
    public void testTypeWithSpecialChars() throws AtlasException {
        HierarchicalTypeDefinition<ClassType> specialTypeDef1 = createClassTypeDef("SpecialTypeDef1", "Typedef with special character",
                ImmutableSet.<String>of(), createRequiredAttrDef("attribute$", DataTypes.STRING_TYPE));

        HierarchicalTypeDefinition<ClassType> specialTypeDef2 = createClassTypeDef("SpecialTypeDef2", "Typedef with special character",
                ImmutableSet.<String>of(), createRequiredAttrDef("attribute%", DataTypes.STRING_TYPE));

        HierarchicalTypeDefinition<ClassType> specialTypeDef3 = createClassTypeDef("SpecialTypeDef3", "Typedef with special character",
                ImmutableSet.<String>of(), createRequiredAttrDef("attribute{", DataTypes.STRING_TYPE));

        HierarchicalTypeDefinition<ClassType> specialTypeDef4 = createClassTypeDef("SpecialTypeDef4", "Typedef with special character",
                ImmutableSet.<String>of(), createRequiredAttrDef("attribute}", DataTypes.STRING_TYPE));

        HierarchicalTypeDefinition<ClassType> specialTypeDef5 = createClassTypeDef("SpecialTypeDef5", "Typedef with special character",
                ImmutableSet.<String>of(), createRequiredAttrDef("attribute$%{}", DataTypes.STRING_TYPE));

        TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
                ImmutableList.<StructTypeDefinition>of(),
                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
                ImmutableList.of(specialTypeDef1, specialTypeDef2, specialTypeDef3, specialTypeDef4, specialTypeDef5));

        Map<String, IDataType> createdTypes = ts.defineTypes(typesDef);
        typeStore.store(ts, ImmutableList.copyOf(createdTypes.keySet()));

        //Validate the updated types
        TypesDef types = typeStore.restore();
        ts.reset();
        ts.defineTypes(types);
    }

    @Test(dependsOnMethods = "testStore")
    public void testTypeUpdate() throws Exception {
        //Add enum value
        String _description = "_description_updated";
        EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", "OrgLevel"+_description, new EnumValue("L1", 1),
                new EnumValue("L2", 2), new EnumValue("L3", 3));

        //Add attribute
        StructTypeDefinition addressDetails =
                createStructTypeDef("Address", createRequiredAttrDef("street", DataTypes.STRING_TYPE),
                        createRequiredAttrDef("city", DataTypes.STRING_TYPE),
                        createOptionalAttrDef("state", DataTypes.STRING_TYPE));

        HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department", "Department"+_description,
            ImmutableSet.<String>of(), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
                new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.OPTIONAL,
                        true, "department"),
                new AttributeDefinition("positions", String.format("map<%s,%s>", DataTypes.STRING_TYPE.getName(), "Person"), Multiplicity.OPTIONAL, false, null));
        TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
                ImmutableList.of(deptTypeDef));

        Map<String, IDataType> typesAdded = ts.updateTypes(typesDef);
        typeStore.store(ts, ImmutableList.copyOf(typesAdded.keySet()));

        verifyEdges();
        
        //Validate the updated types
        TypesDef types = typeStore.restore();
        ts.reset();
        ts.defineTypes(types);

        //Assert new enum value
        EnumType orgLevel = ts.getDataType(EnumType.class, orgLevelEnum.name);
        Assert.assertEquals(orgLevel.name, orgLevelEnum.name);
        Assert.assertEquals(orgLevel.description, orgLevelEnum.description);
        Assert.assertEquals(orgLevel.values().size(), orgLevelEnum.enumValues.length);
        Assert.assertEquals(orgLevel.fromValue("L3").ordinal, 3);

        //Assert new attribute
        StructType addressType = ts.getDataType(StructType.class, addressDetails.typeName);
        Assert.assertEquals(addressType.numFields, 3);
        Assert.assertEquals(addressType.fieldMapping.fields.get("state").dataType(), DataTypes.STRING_TYPE);

        //Updating the definition again shouldn't add another edge
        typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
                ImmutableList.<StructTypeDefinition>of(),
                ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
                ImmutableList.of(deptTypeDef));
        typesAdded = ts.updateTypes(typesDef);
        typeStore.store(ts, ImmutableList.copyOf(typesAdded.keySet()));
        verifyEdges();
    }

    private void verifyEdges() throws RepositoryException {
        // ATLAS-474: verify that type update did not write duplicate edges to the type store.
        if (typeStore instanceof GraphBackedTypeStore) {
            GraphBackedTypeStore gbTypeStore = (GraphBackedTypeStore) typeStore;
            AtlasVertex typeVertex = gbTypeStore.findVertices(Collections.singletonList("Department")).get("Department");
            int edgeCount = countOutgoingEdges(typeVertex, gbTypeStore.getEdgeLabel("Department", "employees"));
            Assert.assertEquals(edgeCount, 1, "Should only be 1 edge for employees attribute on Department type AtlasVertex");
        }
    }

    private int countOutgoingEdges(AtlasVertex typeVertex, String edgeLabel) {

        Iterator<AtlasEdge> outGoingEdgesByLabel = GraphHelper.getInstance().getOutGoingEdgesByLabel(typeVertex, edgeLabel);
        int edgeCount = 0;
        for (; outGoingEdgesByLabel.hasNext();) {
            outGoingEdgesByLabel.next();
            edgeCount++;
        }
        return edgeCount;
    }

    private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException {
        boolean clsTypeFound = false;
        List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
        for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
            if (classType.typeName.equals(typeName)) {
                ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
                Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
                Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
                Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION);
                clsTypeFound = true;
            }
        }
        Assert.assertTrue(clsTypeFound, typeName + " type not restored");
    }
}