DimensionLookupMetaTest.java

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.trans.steps.dimensionlookup;

import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;

import org.apache.commons.lang.StringUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import org.pentaho.di.core.KettleEnvironment;
import org.pentaho.di.core.SQLStatement;
import org.pentaho.di.core.database.Database;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleDatabaseException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.logging.KettleLogStore;
import org.pentaho.di.core.logging.LogChannelInterface;
import org.pentaho.di.core.logging.LogChannelInterfaceFactory;
import org.pentaho.di.core.logging.LoggingObjectInterface;
import org.pentaho.di.core.plugins.PluginRegistry;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaString;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.repository.Repository;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;
import org.pentaho.di.trans.steps.loadsave.LoadSaveTester;
import org.pentaho.di.trans.steps.loadsave.initializer.InitializerInterface;
import org.pentaho.di.trans.steps.loadsave.validator.ArrayLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.DatabaseMetaLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.FieldLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.IntLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.NonZeroIntLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.PrimitiveIntArrayLoadSaveValidator;
import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator;
import org.pentaho.metastore.api.IMetaStore;

public class DimensionLookupMetaTest implements InitializerInterface<StepMetaInterface> {
  LoadSaveTester loadSaveTester;
  Class<DimensionLookupMeta> testMetaClass = DimensionLookupMeta.class;
  private ThreadLocal<DimensionLookupMeta> holdTestingMeta = new ThreadLocal<DimensionLookupMeta>();

  @Before
  public void setUpLoadSave() throws Exception {
    KettleEnvironment.init();
    PluginRegistry.init( true );
    List<String> attributes =
        Arrays.asList( "schemaName", "tableName", "update", "dateField", "dateFrom", "dateTo", "keyField", "keyRename",
            "autoIncrement", "versionField", "commitSize", "useBatchUpdate", "minYear", "maxYear", "techKeyCreation",
            "cacheSize", "usingStartDateAlternative", "startDateAlternative", "startDateFieldName", "preloadingCache", "keyStream",
            "keyLookup", "fieldStream", "fieldLookup", "fieldUpdate", "databaseMeta", "sequenceName" );

    Map<String, String> getterMap = new HashMap<String, String>() {
      {
        put( "useBatchUpdate", "useBatchUpdate" );
      }
    };
    Map<String, String> setterMap = new HashMap<String, String>();

    FieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
        new ArrayLoadSaveValidator<String>( new StringLoadSaveValidator(), 5 );

    Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
    attrValidatorMap.put( "keyStream", stringArrayLoadSaveValidator );
    attrValidatorMap.put( "keyLookup", stringArrayLoadSaveValidator );
    attrValidatorMap.put( "fieldStream", stringArrayLoadSaveValidator );
    attrValidatorMap.put( "fieldLookup", stringArrayLoadSaveValidator );
    // Note - have to use the non-zero int load/save validator here because if "update"
    // is false, code in DimensionLookupMeta replaces "ValueMetaInterface.TYPE_NONE" with
    // ValueMetaInterface.TYPE_STRING. This happens about once out of every 3 or so runs of
    // the test which made it a bit difficult to track down.
    // MB - 5/2016
    attrValidatorMap.put( "fieldUpdate", new FieldUpdateIntArrayLoadSaveValidator( new NonZeroIntLoadSaveValidator(
        DimensionLookupMeta.typeDesc.length ), 5 ) );
    attrValidatorMap.put( "databaseMeta", new DatabaseMetaLoadSaveValidator() );
    attrValidatorMap.put( "startDateAlternative", new IntLoadSaveValidator( DimensionLookupMeta.getStartDateAlternativeCodes().length ) );
    attrValidatorMap.put( "sequenceName", new SequenceNameLoadSaveValidator() );

    Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();

    loadSaveTester =
        new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(),
            getterMap, setterMap, attrValidatorMap, typeValidatorMap, this );
  }

  // Call the allocate method on the LoadSaveTester meta class
  @Override
  public void modify( StepMetaInterface someMeta ) {
    if ( someMeta instanceof DimensionLookupMeta ) {
      ( (DimensionLookupMeta) someMeta ).allocate( 5, 5 );
      // doing this as a work-around for sequenceName validation.
      // Apparently, sequenceName will always be written (getXml),
      // but will only be read if the value of "update" is true.
      // While testing the load/save behavior, there is no sane way
      // to test dependent variables like this (that I could see). So,
      // I'm holding onto the meta, and will have a special load/save handler
      // for sequenceName.
      // MB - 5/2016
      this.holdTestingMeta.set( (DimensionLookupMeta) someMeta );
    }
  }

  @Test
  public void testSerialization() throws KettleException {
    loadSaveTester.testSerialization();
  }

  public static final String databaseXML =
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
      + "<connection>" + "<name>lookup</name>" + "<server>127.0.0.1</server>" + "<type>H2</type>"
      + "<access>Native</access>" + "<database>mem:db</database>" + "<port></port>" + "<username>sa</username>"
      + "<password></password>" + "</connection>";


  @Before
  public void setUp() throws Exception {
    LogChannelInterfaceFactory logChannelInterfaceFactory = mock( LogChannelInterfaceFactory.class );
    LogChannelInterface logChannelInterface = mock( LogChannelInterface.class );
    KettleLogStore.setLogChannelInterfaceFactory( logChannelInterfaceFactory );
    when( logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn(
        logChannelInterface );
  }

  @Test
  public void testGetFields() throws Exception {

    RowMeta extraFields = new RowMeta();
    extraFields.addValueMeta( new ValueMetaString( "field1" ) );

    DatabaseMeta dbMeta = mock( DatabaseMeta.class );

    DimensionLookupMeta meta = spy( new DimensionLookupMeta() );
    meta.setUpdate( false );
    meta.setKeyField( null );
    meta.setFieldLookup( new String[] { "field1" } );
    meta.setFieldStream( new String[] { "" } );
    meta.setDatabaseMeta( dbMeta );
    doReturn( extraFields ).when( meta ).getDatabaseTableFields( (Database) anyObject(), anyString(), anyString() );
    doReturn( mock( LogChannelInterface.class ) ).when( meta ).getLog();

    RowMeta row = new RowMeta();
    try {
      meta.getFields( row, "DimensionLookupMetaTest", new RowMeta[] { row }, null, null, null, null );
    } catch ( Throwable e ) {
      Assert.assertTrue( e.getMessage().contains(
          BaseMessages.getString( DimensionLookupMeta.class, "DimensionLookupMeta.Error.NoTechnicalKeySpecified" ) ) );
    }
  }

  @Test
  public void testUseDefaultSchemaName() throws Exception {
    KettleEnvironment.init();

    String schemaName = "";
    String tableName = "tableName";
    String schemaTable = "default.tableName";
    String keyField = "keyField";

    DatabaseMeta databaseMeta = spy( new DatabaseMeta( databaseXML ) {
      @Override
      public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, boolean use_autoinc ) {
        return "someValue";
      }
    } );
    when( databaseMeta.getQuotedSchemaTableCombination( schemaName, tableName ) ).thenReturn( schemaTable );

    DimensionLookupMeta dlm = new DimensionLookupMeta();
    dlm.setUpdate( true );
    dlm.setDatabaseMeta( databaseMeta );
    dlm.setTableName( tableName );
    dlm.setSchemaName( schemaName );
    dlm.setKeyLookup( new String[] { "keyLookup1", "keyLookup2" } );
    dlm.setKeyStream( new String[] { "keyStream1", "keyStream2" } );
    dlm.setFieldLookup( new String[] { "fieldLookup1", "fieldLookup2" } );
    dlm.setFieldStream( new String[] { "FieldStream1", "FieldStream2" } );
    dlm.setFieldUpdate( new int[] { 1, 2 } );
    dlm.setKeyField( keyField );

    StepMeta stepMeta = mock( StepMeta.class );

    RowMetaInterface rowMetaInterface = mock( RowMetaInterface.class );
    when( rowMetaInterface.size() ).thenReturn( 1 );

    Repository repository = mock( Repository.class );
    IMetaStore metaStore = mock( IMetaStore.class );

    SQLStatement sqlStatement =
        dlm.getSQLStatements( new TransMeta(), stepMeta, rowMetaInterface, repository, metaStore );

    String sql = sqlStatement.getSQL();
    assertEquals( 3, StringUtils.countMatches( sql, schemaTable ) );
  }

  @Test
  public void testProvidesModelerMeta() throws Exception {

    final RowMeta rowMeta = Mockito.mock( RowMeta.class );
    final DimensionLookupMeta dimensionLookupMeta = new DimensionLookupMeta() {
      @Override Database createDatabaseObject() {
        return mock( Database.class );
      }

      @Override protected RowMetaInterface getDatabaseTableFields( Database db, String schemaName, String tableName )
        throws KettleDatabaseException {
        assertEquals( "aSchema", schemaName );
        assertEquals( "aDimTable", tableName );
        return rowMeta;
      }
    };
    dimensionLookupMeta.setFieldLookup( new String[] { "f1", "f2", "f3" } );
    dimensionLookupMeta.setKeyLookup( new String[] {"k1"} );
    dimensionLookupMeta.setFieldStream( new String[] { "s4", "s5", "s6" } );
    dimensionLookupMeta.setKeyStream( new String[] {"ks1"} );
    dimensionLookupMeta.setSchemaName( "aSchema" );
    dimensionLookupMeta.setTableName( "aDimTable" );

    final DimensionLookupData dimensionLookupData = new DimensionLookupData();
    assertEquals( rowMeta, dimensionLookupMeta.getRowMeta( dimensionLookupData ) );
    assertEquals( 4, dimensionLookupMeta.getDatabaseFields().size() );
    assertEquals( "f1", dimensionLookupMeta.getDatabaseFields().get( 0 ) );
    assertEquals( "f2", dimensionLookupMeta.getDatabaseFields().get( 1 ) );
    assertEquals( "f3", dimensionLookupMeta.getDatabaseFields().get( 2 ) );
    assertEquals( "k1", dimensionLookupMeta.getDatabaseFields().get( 3 ) );
    assertEquals( 4, dimensionLookupMeta.getStreamFields().size() );
    assertEquals( "s4", dimensionLookupMeta.getStreamFields().get( 0 ) );
    assertEquals( "s5", dimensionLookupMeta.getStreamFields().get( 1 ) );
    assertEquals( "s6", dimensionLookupMeta.getStreamFields().get( 2 ) );
    assertEquals( "ks1", dimensionLookupMeta.getStreamFields().get( 3 ) );
  }

  // Note - Removed cloneTest since it's covered by the load/save tester

  // Doing this as a work-around for sequenceName validation.
  // Apparently, sequenceName will always be written (getXml),
  // but will only be read if the value of "update" is true (readData).
  // While testing the load/save behavior, there is no sane way
  // to test dependent variables like this (that I could see). So,
  // I'm holding onto the meta in a threadlocal, and have to have
  // this special load/save handler for sequenceName.
  // MB - 5/2016
  public class SequenceNameLoadSaveValidator implements FieldLoadSaveValidator<String> {
    final Random rand = new Random();
    @Override
    public String getTestObject() {
      DimensionLookupMeta dlm = holdTestingMeta.get(); // get the currently-being tested meta
      if ( dlm.isUpdate() ) { // value returned here is dependant on isUpdate()
        return UUID.randomUUID().toString(); // return a string
      } else {
        return null; // Return null if !isUpdate ...
      }
    }

    @Override
    public boolean validateTestObject( String testObject, Object actual ) {
      String another = (String) actual;
      DimensionLookupMeta dlm = holdTestingMeta.get();
      if ( dlm.isUpdate() ) {
        return testObject.equals( another ); // if isUpdate, compare strings
      } else {
        return ( another == null ); // If !isUpdate, another should be null
      }
    }
  }

  public class FieldUpdateIntArrayLoadSaveValidator extends PrimitiveIntArrayLoadSaveValidator {

    public FieldUpdateIntArrayLoadSaveValidator( FieldLoadSaveValidator<Integer> fieldValidator ) {
      this( fieldValidator, null );
    }

    public FieldUpdateIntArrayLoadSaveValidator( FieldLoadSaveValidator<Integer> fieldValidator, Integer elements ) {
      super( fieldValidator, elements );
    }

    @Override
    public int[] getTestObject() {
      DimensionLookupMeta dlm = holdTestingMeta.get();
      int[] testObject = super.getTestObject();
      if ( !dlm.isUpdate() ) {
        dlm.setReturnType( testObject );
      }
      return testObject;
    }
  }

}