KettleDatabaseRepositoryTest.java

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.repository;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

import junit.framework.TestCase;

import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.FileSelectInfo;
import org.apache.commons.vfs2.FileSelector;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.job.JobMeta;
import org.pentaho.di.repository.kdr.KettleDatabaseRepository;
import org.pentaho.di.repository.kdr.KettleDatabaseRepositoryCreationHelper;
import org.pentaho.di.repository.kdr.KettleDatabaseRepositoryMeta;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.metastore.api.IMetaStore;

public class KettleDatabaseRepositoryTest extends TestCase {

  protected KettleDatabaseRepositoryMeta repositoryMeta;
  protected KettleDatabaseRepository repository;
  protected RepositoryDirectoryInterface tree;

  public void testDatabaseRepository() throws Exception {

    KettleEnvironment.init();
    String filename = File.createTempFile( "kdrtest", "" ).getAbsolutePath();
    System.out.println( "Using file '" + filename + "' as a H2 database repository" );

    try {
      DatabaseMeta databaseMeta = new DatabaseMeta( "H2Repo", "H2", "JDBC", null, filename, null, null, null );
      repositoryMeta =
        new KettleDatabaseRepositoryMeta( "KettleDatabaseRepository", "H2Repo", "H2 Repository", databaseMeta );
      repository = new KettleDatabaseRepository();
      repository.init( repositoryMeta );
      repository.connectionDelegate.connect( true, true );
      KettleDatabaseRepositoryCreationHelper helper = new KettleDatabaseRepositoryCreationHelper( repository );
      helper.createRepositorySchema( null, false, new ArrayList<String>(), false );
      repository.disconnect();

      // Test connecting...
      //
      repository.connect( "admin", "admin" );
      assertTrue( repository.isConnected() );
      
      // Test database save
      DatabaseMeta dataBaseForSave = new DatabaseMeta("H2Test", "H2", "JDBC", null, filename, null, null, null );   
      repository.save( dataBaseForSave, "User creates new database" );
      //load new database from repository by generated id on save step
      DatabaseMeta loadedDataBase = repository.loadDatabaseMeta( dataBaseForSave.getObjectId(), "User creates new database" );
      
      assertEquals("Database object before save and after load form database is diffenert", dataBaseForSave, loadedDataBase );
      
      // Test loading the directory tree
      tree = repository.loadRepositoryDirectoryTree();
      assertNotNull( tree );

      // Test directory creation
      //
      RepositoryDirectoryInterface fooDirectory = repository.createRepositoryDirectory( tree, "foo" );
      RepositoryDirectoryInterface barDirectory = repository.createRepositoryDirectory( fooDirectory, "bar" );
      RepositoryDirectoryInterface samplesDirectory =
        repository.createRepositoryDirectory( fooDirectory, "samples" );

      // Test directory path lookup
      RepositoryDirectoryInterface checkBar = tree.findDirectory( "/foo/bar" );
      assertNotNull( checkBar );
      assertTrue( checkBar.equals( barDirectory ) );

      // Save all the transformations samples.
      //
      verifyTransformationSamples( samplesDirectory );
      verifyJobSamples( samplesDirectory );

      // Verify metastore functionality
      //
      IMetaStore metaStore = repository.getMetaStore();
      KettleMetaStoreTestBase testBase = new KettleMetaStoreTestBase();
      testBase.testFunctionality( metaStore );

      // Finally test disconnecting
      repository.disconnect();
      assertFalse( repository.isConnected() );

    } catch ( Exception e ) {
      e.printStackTrace();
      throw new KettleException( "Error during database repository unit testing", e );
    } finally {
      // Remove the H2 database file
      //
      assertTrue( new File( filename + ".h2.db" ).delete() );
      assertTrue( new File( filename + ".trace.db" ).delete() );
    }
  }

  protected void verifyTransformationSamples( RepositoryDirectoryInterface samplesDirectory ) throws Exception {
    File transSamplesFolder = new File( "samples/transformations/" );
    String[] files = transSamplesFolder.list( new FilenameFilter() {
      public boolean accept( File dir, String name ) {
        return name.endsWith( ".ktr" ) && !name.contains( "HL7" );
      }
    } );
    Arrays.sort( files );

    for ( String file : files ) {
      String transFilename = transSamplesFolder.getAbsolutePath() + "/" + file;
      System.out.println( "Storing/Loading/validating transformation '" + transFilename + "'" );

      // Load the TransMeta object...
      //
      TransMeta transMeta = new TransMeta( transFilename );
      if ( Utils.isEmpty( transMeta.getName() ) ) {
        transMeta.setName( Const.createName( file ) );
      }

      // Save it in the repository in the samples folder
      //
      transMeta.setRepositoryDirectory( samplesDirectory );
      repository.save( transMeta, "unit testing", null, true );
      assertNotNull( transMeta.getObjectId() );

      // Load it back up again...
      //
      TransMeta repTransMeta = repository.loadTransformation( transMeta.getObjectId(), null );
      String oneXml = repTransMeta.getXML();

      // Save & load it again
      //
      repository.save( transMeta, "unit testing", null, true );
      repTransMeta = repository.loadTransformation( transMeta.getObjectId(), null );
      String twoXml = repTransMeta.getXML();

      // The XML needs to be identical after loading
      //
      // storeFile(oneXml, "/tmp/one.ktr");
      // storeFile(twoXml, "/tmp/two.ktr");

      assertEquals( oneXml, twoXml );
    }

    // Verify the number of stored files, see if we can find them all again.
    //
    System.out.println( "Stored "
      + files.length + " transformation samples in folder " + samplesDirectory.getPath() );
    String[] transformationNames = repository.getTransformationNames( samplesDirectory.getObjectId(), false );
    assertEquals( files.length, transformationNames.length );
  }

  protected void verifyJobSamples( RepositoryDirectoryInterface samplesDirectory ) throws Exception {
    FileObject jobSamplesFolder = KettleVFS.getFileObject( "samples/jobs/" );
    FileObject[] files = jobSamplesFolder.findFiles( new FileSelector() {

      @Override
      public boolean traverseDescendents( FileSelectInfo arg0 ) throws Exception {
        return true;
      }

      @Override
      public boolean includeFile( FileSelectInfo info ) throws Exception {
        return info.getFile().getName().getExtension().equalsIgnoreCase( "kjb" );
      }
    } );

    List<FileObject> filesList = Arrays.asList( files );
    Collections.sort( filesList, new Comparator<FileObject>() {
      @Override
      public int compare( FileObject o1, FileObject o2 ) {
        return o1.getName().getPath().compareTo( o2.getName().getPath() );
      }
    } );

    // test the storage of jobMeta attributes in the Kettle DB Repo
    if ( filesList.size() > 0 ) {
      FileObject file = filesList.get( 0 );
      String jobFilename = file.getName().getPath();
      System.out.println( "Storing/Loading/validating job attributes" );

      // Load the JobMeta object...
      //
      JobMeta jobMeta = new JobMeta( jobFilename, repository );
      // set some attributes
      jobMeta.setAttribute( "group", "key", "value" );
      jobMeta.setAttribute( "test-group", "test-key-1", "test-value" );
      jobMeta.setAttribute( "test-group", "test-key-2", "test-value" );
      jobMeta.setAttribute( "test-group", "test-key-3", "test-value-3" );

      // Save it in the repository in the samples folder
      //
      jobMeta.setRepositoryDirectory( samplesDirectory );
      repository.save( jobMeta, "unit testing" );
      assertNotNull( jobMeta.getObjectId() );

      // Load it back up again...
      //
      JobMeta repJobMeta = repository.loadJob( jobMeta.getObjectId(), null );
      String value = repJobMeta.getAttribute( "group", "key" );
      String value1 = repJobMeta.getAttribute( "test-group", "test-key-1" );
      String value2 = repJobMeta.getAttribute( "test-group", "test-key-2" );
      String value3 = repJobMeta.getAttribute( "test-group", "test-key-3" );
      assertEquals( "value", value );
      assertEquals( "test-value", value1 );
      assertEquals( "test-value", value2 );
      assertEquals( "test-value-3", value3 );
    }
    
    for ( FileObject file : filesList ) {
      String jobFilename = file.getName().getPath();
      System.out.println( "Storing/Loading/validating job '" + jobFilename + "'" );

      // Load the JobMeta object...
      //
      JobMeta jobMeta = new JobMeta( jobFilename, repository );
      if ( Utils.isEmpty( jobMeta.getName() ) ) {
        jobMeta.setName( Const.createName( file.getName().getBaseName() ) );
      }

      // Save it in the repository in the samples folder
      //
      jobMeta.setRepositoryDirectory( samplesDirectory );
      repository.save( jobMeta, "unit testing" );
      assertNotNull( jobMeta.getObjectId() );

      // Load it back up again...
      //
      JobMeta repJobMeta = repository.loadJob( jobMeta.getObjectId(), null );
      String oneXml = repJobMeta.getXML();

      // Save & load it again
      //
      repository.save( jobMeta, "unit testing" );
      repJobMeta = repository.loadJob( jobMeta.getObjectId(), null );
      String twoXml = repJobMeta.getXML();

      // The XML needs to be identical after loading
      //
      // storeFile(oneXml, "/tmp/one.ktr");
      // storeFile(twoXml, "/tmp/two.ktr");
      //
      assertEquals( oneXml, twoXml );
    }

    // Verify the number of stored files, see if we can find them all again.
    //
    System.out.println( "Stored " + files.length + " job samples in folder " + samplesDirectory.getPath() );
    String[] jobNames = repository.getJobNames( samplesDirectory.getObjectId(), false );
    assertEquals( files.length, jobNames.length );
  }

  protected void storeFile( String xml, String filename ) throws Exception {
    File file = new File( filename );
    FileOutputStream fos = new FileOutputStream( file );

    fos.write( XMLHandler.getXMLHeader( Const.XML_ENCODING ).getBytes( Const.XML_ENCODING ) );
    fos.write( xml.getBytes( Const.XML_ENCODING ) );
    fos.close();

  }

}