PurRepositoryMetaStore.java

/*!
 * Copyright 2010 - 2016 Pentaho Corporation.  All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package org.pentaho.di.repository.pur.metastore;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.repository.pur.PurRepository;
import org.pentaho.metastore.api.IMetaStore;
import org.pentaho.metastore.api.IMetaStoreAttribute;
import org.pentaho.metastore.api.IMetaStoreElement;
import org.pentaho.metastore.api.IMetaStoreElementType;
import org.pentaho.metastore.api.exceptions.MetaStoreDependenciesExistsException;
import org.pentaho.metastore.api.exceptions.MetaStoreElementTypeExistsException;
import org.pentaho.metastore.api.exceptions.MetaStoreException;
import org.pentaho.metastore.api.exceptions.MetaStoreNamespaceExistsException;
import org.pentaho.metastore.stores.memory.MemoryMetaStore;
import org.pentaho.platform.api.repository2.unified.IUnifiedRepository;
import org.pentaho.platform.api.repository2.unified.RepositoryFile;
import org.pentaho.platform.api.repository2.unified.RepositoryFileAcl;
import org.pentaho.platform.api.repository2.unified.RepositoryRequest;
import org.pentaho.platform.api.repository2.unified.data.node.DataNode;
import org.pentaho.platform.api.repository2.unified.data.node.DataProperty;
import org.pentaho.platform.api.repository2.unified.data.node.NodeRepositoryFileData;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

/**
 * Please note that for this class to work, the supplied PurRepository needs to be connected to the server.
 * 
 * @author matt
 */
public class PurRepositoryMetaStore extends MemoryMetaStore implements IMetaStore {

  public static final String ELEMENT_TYPE_DETAILS_FILENAME = "ElementTypeDetails";

  protected static final String PROP_NAME = "NAME";
  protected static final String PROP_ELEMENT_TYPE_NAME = "element_type_name";
  protected static final String PROP_ELEMENT_CHILDREN = "element_children";

  protected static final String PROP_ELEMENT_TYPE_DESCRIPTION = "element_type_description"; //$NON-NLS-1$

  protected static final String METASTORE_FOLDER_PATH = "/etc/metastore";

  protected PurRepository repository;
  protected IUnifiedRepository pur;

  /**
   * This is the folder where the namespaces folders are stored
   */
  protected RepositoryFile namespacesFolder;

  public PurRepositoryMetaStore( PurRepository repository ) throws KettleException {
    this.repository = repository;
    this.pur = repository.getPur();

    namespacesFolder = pur.getFile( METASTORE_FOLDER_PATH );
    if ( namespacesFolder == null ) {
      throw new KettleException( METASTORE_FOLDER_PATH + " folder is not available" );
    }
  }

  @Override
  public String getName() {
    return repository.getRepositoryMeta().getName();
  }

  @Override
  public String getDescription() {
    return repository.getRepositoryMeta().getDescription();
  }

  // The namespaces

  @Override
  public void createNamespace( String namespace ) throws MetaStoreException {
    if ( namespaceExists( namespace ) ) {
      throw new MetaStoreNamespaceExistsException( "Namespace '" + namespace
          + "' can not be created, it already exists" );
    }
    RepositoryFile namespaceFolder = new RepositoryFile.Builder( namespace ).folder( true ).versioned( false ).build();

    pur.createFolder( namespacesFolder.getId(), namespaceFolder, "Created namespace" );
  }

  @Override
  public boolean namespaceExists( String namespace ) throws MetaStoreException {
    return getNamespaceRepositoryFile( namespace ) != null;
  }

  @Override
  public void deleteNamespace( String namespace ) throws MetaStoreException {
    RepositoryFile namespaceFile = getNamespaceRepositoryFile( namespace );
    if ( namespaceFile == null ) {
      return; // already gone.
    }
    List<RepositoryFile> children = getChildren( namespaceFile.getId() );
    if ( children == null || children.isEmpty() ) {
      // Delete the file, there are no children.
      //
      pur.deleteFile( namespaceFile.getId(), true, "Delete namespace" );
    } else {
      // Dependencies exists, throw an exception.
      //
      List<String> elementTypeIds = new ArrayList<String>();
      for ( RepositoryFile child : children ) {
        elementTypeIds.add( child.getId().toString() );
      }
      throw new MetaStoreDependenciesExistsException( elementTypeIds, "Namespace '" + namespace
          + " can not be deleted because it is not empty" );
    }
  }

  @Override
  public List<String> getNamespaces() throws MetaStoreException {
    List<String> namespaces = new ArrayList<String>();
    List<RepositoryFile> children = getChildren( namespacesFolder.getId() );
    for ( RepositoryFile child : children ) {
      namespaces.add( child.getName() );
    }
    return namespaces;
  }

  // The element types

  @Override
  public void createElementType( String namespace, IMetaStoreElementType elementType ) throws MetaStoreException {

    RepositoryFile namespaceFile = validateNamespace( namespace );

    IMetaStoreElementType existingType = getElementTypeByName( namespace, elementType.getName() );
    if ( existingType != null ) {
      throw new MetaStoreElementTypeExistsException( Collections.singletonList( existingType ),
          "Can not create element type with id '" + elementType.getId() + "' because it already exists" );
    }

    RepositoryFile elementTypeFile =
        new RepositoryFile.Builder( elementType.getName() ).folder( true ).versioned( false ).build();

    RepositoryFile folder = pur.createFolder( namespaceFile.getId(), elementTypeFile, null );
    elementType.setId( folder.getId().toString() );

    // In this folder there is a hidden file which contains the description
    // and the other future properties of the element type
    //
    RepositoryFile detailsFile =
        new RepositoryFile.Builder( ELEMENT_TYPE_DETAILS_FILENAME ).folder( false ).title(
            ELEMENT_TYPE_DETAILS_FILENAME ).description( elementType.getDescription() ).hidden( true ).build();

    DataNode dataNode = getElementTypeDataNode( elementType );

    pur.createFile( folder.getId(), detailsFile, new NodeRepositoryFileData( dataNode ), null );

    elementType.setMetaStoreName( getName() );
  }

  @Override
  public synchronized void updateElementType( String namespace, IMetaStoreElementType elementType )
    throws MetaStoreException {
    RepositoryFile folder = getElementTypeRepositoryFolder( namespace, elementType );

    RepositoryFile detailsFile = findChildByName( folder.getId(), ELEMENT_TYPE_DETAILS_FILENAME, true );

    DataNode dataNode = getElementTypeDataNode( elementType );

    pur.updateFile( detailsFile, new NodeRepositoryFileData( dataNode ), null );

    elementType.setMetaStoreName( getName() );
  }

  private DataNode getElementTypeDataNode( IMetaStoreElementType elementType ) {
    DataNode dataNode = new DataNode( ELEMENT_TYPE_DETAILS_FILENAME );
    dataNode.setProperty( PROP_ELEMENT_TYPE_DESCRIPTION, elementType.getDescription() );
    dataNode.setProperty( PROP_ELEMENT_TYPE_NAME, elementType.getName() );
    dataNode.setProperty( PROP_NAME, elementType.getName() );
    return dataNode;
  }

  @Override
  public IMetaStoreElementType getElementType( String namespace, String elementTypeId ) throws MetaStoreException {
    RepositoryFile elementTypeFolder = pur.getFileById( elementTypeId );
    if ( elementTypeFolder == null ) {
      return null;
    }
    IMetaStoreElementType elementType = newElementType( namespace );
    elementType.setId( elementTypeFolder.getId().toString() );
    elementType.setName( elementTypeFolder.getName() );

    RepositoryFile detailsFile = findChildByName( elementTypeFolder.getId(), ELEMENT_TYPE_DETAILS_FILENAME, true );
    if ( detailsFile != null ) {
      NodeRepositoryFileData data = pur.getDataForRead( detailsFile.getId(), NodeRepositoryFileData.class );
      DataProperty property = data.getNode().getProperty( "element_type_description" );
      if ( property != null ) {
        elementType.setDescription( property.getString() );
      }
    }
    return elementType;
  }

  @Override
  public List<IMetaStoreElementType> getElementTypes( String namespace ) throws MetaStoreException {
    List<IMetaStoreElementType> elementTypes = new ArrayList<IMetaStoreElementType>();

    RepositoryFile namespaceFile = validateNamespace( namespace );
    List<RepositoryFile> children = getChildren( namespaceFile.getId() );
    for ( RepositoryFile child : children ) {
      if ( !child.isHidden() ) {
        elementTypes.add( getElementType( namespace, child.getId().toString() ) );
      }
    }

    return elementTypes;
  }

  @Override
  public IMetaStoreElementType getElementTypeByName( String namespace, String elementTypeName )
    throws MetaStoreException {
    RepositoryFile file = getElementTypeRepositoryFileByName( namespace, elementTypeName );
    if ( file == null ) {
      return null;
    }

    return getElementType( namespace, file.getId().toString() );
  }

  @Override
  public List<String> getElementTypeIds( String namespace ) throws MetaStoreException {
    List<String> ids = new ArrayList<String>();

    for ( IMetaStoreElementType type : getElementTypes( namespace ) ) {
      ids.add( type.getId() );
    }

    return ids;
  }

  @Override
  public void deleteElementType( String namespace, IMetaStoreElementType elementType ) throws MetaStoreException {

    RepositoryFile namespaceRepositoryFile = validateNamespace( namespace );

    RepositoryFile elementTypeFile = findChildByName( namespaceRepositoryFile.getId(), elementType.getName() );
    List<RepositoryFile> children = getChildren( elementTypeFile.getId() );
    removeHiddenFilesFromList( children );

    if ( children.isEmpty() ) {
      pur.deleteFile( elementTypeFile.getId(), true, null );
    } else {
      List<String> ids = getElementIds( namespace, elementType );
      throw new MetaStoreDependenciesExistsException( ids, "Can't delete element type with name '"
          + elementType.getName() + "' because it is not empty" );
    }
  }

  protected void removeHiddenFilesFromList( List<RepositoryFile> children ) {

    for ( Iterator<RepositoryFile> it = children.iterator(); it.hasNext(); ) {
      RepositoryFile child = it.next();
      if ( child.isHidden() ) {
        it.remove();
      }
    }

  }

  // The elements

  public void createElement( String namespace, IMetaStoreElementType elementType, IMetaStoreElement element )
    throws MetaStoreException {
    RepositoryFile elementTypeFolder = validateElementTypeRepositoryFolder( namespace, elementType );

    RepositoryFile elementFile =
        new RepositoryFile.Builder( PurRepository.checkAndSanitize( element.getName() ) ).title( element.getName() )
            .versioned( false ).build();

    DataNode elementDataNode = new DataNode( PurRepository.checkAndSanitize( element.getName() ) );
    elementToDataNode( element, elementDataNode );

    RepositoryFile createdFile =
        pur.createFile( elementTypeFolder.getId(), elementFile, new NodeRepositoryFileData( elementDataNode ), null );
    element.setId( createdFile.getId().toString() );

    // Verify existence.
    if ( pur.getFileById( createdFile.getId() ) == null ) {
      throw new RuntimeException( "Unable to verify creation of element '" + element.getName() + "' in folder: "
          + elementTypeFolder.getPath() );
    }
  }

  protected void elementToDataNode( IMetaStoreElement element, DataNode elementDataNode ) {
    elementDataNode.setProperty( PROP_NAME, element.getName() );
    DataNode childrenNode = elementDataNode.addNode( PROP_ELEMENT_CHILDREN );
    if ( Utils.isEmpty( element.getId() ) ) {
      element.setId( element.getName() );
    }
    attributeToDataNode( element, childrenNode );
  }

  protected void dataNodeToElement( DataNode dataNode, IMetaStoreElement element ) throws MetaStoreException {
    DataProperty nameProperty = dataNode.getProperty( PROP_NAME );
    if ( nameProperty != null ) {
      element.setName( nameProperty.getString() );
    }
    DataNode childrenNode = dataNode.getNode( PROP_ELEMENT_CHILDREN );
    dataNodeToAttribute( childrenNode, element );
  }

  @Override
  public synchronized void updateElement( String namespace, IMetaStoreElementType elementType, String elementId,
      IMetaStoreElement element ) throws MetaStoreException {

    // verify that the element type belongs to this meta store
    //
    if ( elementType.getMetaStoreName() == null || !elementType.getName().equals( getName() ) ) {
      String elementTypeName = elementType.getName();
      elementType = getElementTypeByName( namespace, elementTypeName );
      if ( elementType == null ) {
        throw new MetaStoreException( "The element type '" + elementTypeName
            + "' could not be found in the meta store in which you are updating." );
      }
    }

    RepositoryFile existingFile = pur.getFileById( elementId );
    if ( existingFile == null ) {
      throw new MetaStoreException( "The element to update with id " + elementId + " could not be found in the store" );
    }

    DataNode elementDataNode = new DataNode( PurRepository.checkAndSanitize( element.getName() ) );
    elementToDataNode( element, elementDataNode );

    RepositoryFile updatedFile = pur.updateFile( existingFile, new NodeRepositoryFileData( elementDataNode ), null );
    element.setId( updatedFile.getId().toString() );
  }

  @Override
  public IMetaStoreElement getElement( String namespace, IMetaStoreElementType elementType, String elementId )
    throws MetaStoreException {
    NodeRepositoryFileData data = pur.getDataForRead( elementId, NodeRepositoryFileData.class );
    if ( data == null ) {
      return null;
    }

    IMetaStoreElement element = newElement();
    element.setId( elementId );
    element.setElementType( elementType );
    DataNode dataNode = data.getNode();
    dataNodeToElement( dataNode, element );

    return element;
  }

  @Override
  public List<IMetaStoreElement> getElements( String namespace, IMetaStoreElementType elementType )
    throws MetaStoreException {
    List<IMetaStoreElement> elements = new ArrayList<IMetaStoreElement>();

    RepositoryFile typeFolder = validateElementTypeRepositoryFolder( namespace, elementType );
    List<RepositoryFile> children = getChildren( typeFolder.getId() );
    removeHiddenFilesFromList( children );
    for ( RepositoryFile child : children ) {
      IMetaStoreElement element = getElement( namespace, elementType, child.getId().toString() );
      elements.add( element );
    }

    return elements;
  }

  @Override
  public IMetaStoreElement getElementByName( String namespace, IMetaStoreElementType elementType, String name )
    throws MetaStoreException {
    for ( IMetaStoreElement element : getElements( namespace, elementType ) ) {
      if ( element.getName().equals( name ) ) {
        return element;
      }
    }
    return null;
  }

  @Override
  public List<String> getElementIds( String namespace, IMetaStoreElementType elementType ) throws MetaStoreException {
    RepositoryFile folder = validateElementTypeRepositoryFolder( namespace, elementType );
    List<RepositoryFile> children = getChildren( folder.getId() );
    removeHiddenFilesFromList( children );
    List<String> ids = new ArrayList<String>();
    for ( RepositoryFile child : children ) {
      ids.add( child.getId().toString() );
    }
    return ids;
  }

  @Override
  public void deleteElement( String namespace, IMetaStoreElementType elementType, String elementId )
    throws MetaStoreException {

    pur.deleteFile( elementId, true, null );

  }

  protected void attributeToDataNode( IMetaStoreAttribute attribute, DataNode dataNode ) {
    String id = attribute.getId();
    Object value = attribute.getValue();
    if ( id == null ) {
      throw new NullPointerException();
    } else if ( value != null ) {
      if ( value instanceof Double ) {
        dataNode.setProperty( id, (Double) value );
      } else if ( value instanceof Date ) {
        dataNode.setProperty( id, (Date) value );
      } else if ( value instanceof Long ) {
        dataNode.setProperty( id, (Long) value );
      } else {
        dataNode.setProperty( id, value.toString() );
      }
    }

    for ( IMetaStoreAttribute child : attribute.getChildren() ) {
      DataNode subNode = new DataNode( child.getId() );
      attributeToDataNode( child, subNode );
      dataNode.addNode( subNode );
    }
  }

  protected void dataNodeToAttribute( DataNode dataNode, IMetaStoreAttribute attribute ) throws MetaStoreException {
    for ( DataProperty dataProperty : dataNode.getProperties() ) {
      Object value;
      switch ( dataProperty.getType() ) {
        case DATE:
          value = ( dataProperty.getDate() );
          break;
        case DOUBLE:
          value = ( dataProperty.getDouble() );
          break;
        case LONG:
          value = ( dataProperty.getLong() );
          break;
        case STRING:
          value = ( dataProperty.getString() );
          break;
        default:
          continue;
      }

      // Backwards Compatibility
      if ( dataProperty.getName().equals( dataNode.getName() ) ) {
        attribute.setValue( value );
      }
      attribute.addChild( newAttribute( dataProperty.getName(), value ) );
    }

    for ( DataNode subNode : dataNode.getNodes() ) {
      IMetaStoreAttribute subAttr = newAttribute( subNode.getName(), null );
      dataNodeToAttribute( subNode, subAttr );
      attribute.addChild( subAttr );
    }
  }

  protected RepositoryFile validateNamespace( String namespace ) throws MetaStoreException {
    RepositoryFile namespaceFile = getNamespaceRepositoryFile( namespace );
    if ( namespaceFile == null ) {
      throw new MetaStoreException( "Namespace '" + namespace + " doesn't exist in the repository" );
    }
    return namespaceFile;
  }

  protected RepositoryFile validateElementTypeRepositoryFolder( String namespace, IMetaStoreElementType elementType )
    throws MetaStoreException {
    // The element type needs to be known in this repository, we need to have a match by ID
    //
    RepositoryFile elementTypeFolder = pur.getFileById( elementType.getId() );
    if ( elementTypeFolder == null ) {
      StringBuilder builder = new StringBuilder();
      builder.append( namespacesFolder.getPath() ).append( Const.CR );
      String available = getMetaStoreFolders( builder, namespacesFolder, 0 );
      throw new MetaStoreException( "The element type with name '" + elementType.getName()
          + " doesn't exist in namespace '" + namespace + "'." + Const.CR + "Available nodes:" + Const.CR + available );
    }
    return elementTypeFolder;
  }

  private String getMetaStoreFolders( StringBuilder builder, RepositoryFile folder, int level ) {
    String spaces = Const.rightPad( " ", level * 2 );
    builder.append( spaces );
    if ( folder.isFolder() ) {
      builder.append( "/" );
    }
    builder.append( folder.getName() ).append( Const.CR );
    for ( RepositoryFile file : getChildren( folder.getId() ) ) {
      getMetaStoreFolders( builder, file, level + 1 );
    }
    return builder.toString();
  }

  protected RepositoryFile getNamespaceRepositoryFile( String namespace ) {
    return findChildByName( namespacesFolder.getId(), namespace );
  }

  protected RepositoryFile getElementTypeRepositoryFolder( String namespace, IMetaStoreElementType elementType )
    throws MetaStoreException {
    RepositoryFile namespaceRepositoryFile = validateNamespace( namespace );
    return findChildByName( namespaceRepositoryFile.getId(), elementType.getName() );
  }

  protected RepositoryFile getElementTypeRepositoryFileByName( String namespace, String elementTypeName ) {
    RepositoryFile namespaceFolder = getNamespaceRepositoryFile( namespace );
    if ( namespace == null ) {
      return null;
    }
    return findChildByName( namespaceFolder.getId(), elementTypeName );
  }

  protected RepositoryFile findChildByName( Serializable folderId, String childName, boolean showHiddenFiles ) {
    for ( RepositoryFile child : getChildren( folderId, showHiddenFiles ) ) {
      if ( child.getName().equals( childName ) ) {
        return child;
      }
    }
    return null;
  }

  protected RepositoryFile findChildByName( Serializable folderId, String childName ) {
    return findChildByName( folderId, childName, false );
  }

  protected RepositoryFileAcl getAcls() {
    return null; // new RepositoryFileAcl.Builder(RepositoryFileAcl.Builder).entriesInheriting(true).build()
  }


  @Override
  public IMetaStoreElementType newElementType( String namespace ) throws MetaStoreException {
    IMetaStoreElementType elementType = super.newElementType( namespace );
    elementType.setMetaStoreName( getName() );
    return elementType;
  }

  private List<RepositoryFile> getChildren( Serializable folderId ) {
    return getChildren( folderId, false );
  }

  private List<RepositoryFile> getChildren( Serializable folderId, boolean showHiddenFiles ) {
    return pur.getChildren( new RepositoryRequest( folderId.toString(), showHiddenFiles, -1, null ) );
  }

}