S3CsvInputDialog.java

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2016 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/
package org.pentaho.di.trans.steps.s3csvinput;

import java.io.IOException;

import java.io.InputStream;
import java.io.InputStreamReader;
import java.text.DecimalFormat;

import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.events.ModifyEvent;
import org.eclipse.swt.events.ModifyListener;
import org.eclipse.swt.events.SelectionAdapter;
import org.eclipse.swt.events.SelectionEvent;
import org.eclipse.swt.events.ShellAdapter;
import org.eclipse.swt.events.ShellEvent;
import org.eclipse.swt.layout.FormAttachment;
import org.eclipse.swt.layout.FormData;
import org.eclipse.swt.layout.FormLayout;
import org.eclipse.swt.widgets.Button;
import org.eclipse.swt.widgets.Control;
import org.eclipse.swt.widgets.Display;
import org.eclipse.swt.widgets.Event;
import org.eclipse.swt.widgets.Label;
import org.eclipse.swt.widgets.Listener;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.swt.widgets.TableItem;
import org.eclipse.swt.widgets.Text;
import org.jets3t.service.S3Service;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.TransPreviewFactory;
import org.pentaho.di.trans.step.BaseStepMeta;
import org.pentaho.di.trans.step.StepDialogInterface;
import org.pentaho.di.trans.steps.textfileinput.TextFileInput;
import org.pentaho.di.trans.steps.textfileinput.TextFileInputField;
import org.pentaho.di.trans.steps.textfileinput.TextFileInputMeta;
import org.pentaho.di.ui.core.dialog.EnterNumberDialog;
import org.pentaho.di.ui.core.dialog.EnterSelectionDialog;
import org.pentaho.di.ui.core.dialog.EnterTextDialog;
import org.pentaho.di.ui.core.dialog.ErrorDialog;
import org.pentaho.di.ui.core.dialog.PreviewRowsDialog;
import org.pentaho.di.ui.core.widget.ColumnInfo;
import org.pentaho.di.ui.core.widget.ComboValuesSelectionListener;
import org.pentaho.di.ui.core.widget.TableView;
import org.pentaho.di.ui.core.widget.TextVar;
import org.pentaho.di.ui.core.widget.PasswordTextVar;
import org.pentaho.di.ui.trans.dialog.TransPreviewProgressDialog;
import org.pentaho.di.ui.trans.step.BaseStepDialog;
import org.pentaho.di.ui.trans.steps.textfileinput.TextFileCSVImportProgressDialog;

public class S3CsvInputDialog extends BaseStepDialog implements StepDialogInterface {
  private S3CsvInputMeta inputMeta;

  private TextVar      wAccessKey;
  private TextVar      wSecretKey;
  private TextVar      wBucket;
  private Button       wbBucket; // browse for a bucket.
  private TextVar      wFilename;
  private CCombo       wFilenameField;
  private Button       wbbFilename; // Browse for a file
  private Button       wIncludeFilename;
  private TextVar      wRowNumField;
  private Button       wbDelimiter;
  private TextVar      wDelimiter;
  private TextVar      wEnclosure;
  private TextVar      wMaxLineSize;
  private Button       wLazyConversion;
  private Button       wHeaderPresent;

  private TableView    wFields;

  private boolean isReceivingInput;
  private Button wRunningInParallel;

  public S3CsvInputDialog( Shell parent, Object in, TransMeta tr, String sname ) {
    super( parent, (BaseStepMeta) in, tr, sname );
    inputMeta = (S3CsvInputMeta) in;
  }

  @Override
  public String open() {
    Shell parent = getParent();
    Display display = parent.getDisplay();

    shell = new Shell( parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX );
    props.setLook( shell );
    setShellImage( shell, inputMeta );

    ModifyListener lsMod = new ModifyListener() {
      @Override
      public void modifyText( ModifyEvent e ) {
        inputMeta.setChanged();
      }
    };
    changed = inputMeta.hasChanged();

    FormLayout formLayout = new FormLayout();
    formLayout.marginWidth  = Const.FORM_MARGIN;
    formLayout.marginHeight = Const.FORM_MARGIN;

    shell.setLayout( formLayout );
    shell.setText( Messages.getString( "S3CsvInputDialog.Shell.Title" ) ); //$NON-NLS-1$

    int middle = props.getMiddlePct();
    int margin = Const.MARGIN;

    // Step name line
    //
    wlStepname = new Label( shell, SWT.RIGHT );
    wlStepname.setText( Messages.getString( "S3CsvInputDialog.Stepname.Label" ) ); //$NON-NLS-1$
    props.setLook( wlStepname );
    fdlStepname = new FormData();
    fdlStepname.left = new FormAttachment( 0, 0 );
    fdlStepname.right = new FormAttachment( middle, -margin );
    fdlStepname.top = new FormAttachment( 0, margin );
    wlStepname.setLayoutData( fdlStepname );
    wStepname = new Text( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wStepname );
    wStepname.addModifyListener( lsMod );
    fdStepname = new FormData();
    fdStepname.left = new FormAttachment( middle, 0 );
    fdStepname.top = new FormAttachment( 0, margin );
    fdStepname.right = new FormAttachment( 100, 0 );
    wStepname.setLayoutData( fdStepname );
    Control lastControl = wStepname;

    // Access key
    Label wlAccessKey = new Label( shell, SWT.RIGHT );
    wlAccessKey.setText( Messages.getString( "S3CsvInputDialog.AccessKey.Label" ) ); //$NON-NLS-1$
    props.setLook( wlAccessKey );
    FormData fdlAccessKey = new FormData();
    fdlAccessKey.top = new FormAttachment( lastControl, margin );
    fdlAccessKey.left = new FormAttachment( 0, 0 );
    fdlAccessKey.right = new FormAttachment( middle, -margin );
    wlAccessKey.setLayoutData( fdlAccessKey );

    wAccessKey = new PasswordTextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );

    props.setLook( wAccessKey );
    wAccessKey.addModifyListener( lsMod );
    FormData fdAccessKey = new FormData();
    fdAccessKey.top = new FormAttachment( lastControl, margin );
    fdAccessKey.left = new FormAttachment( middle, 0 );
    fdAccessKey.right = new FormAttachment( 100, 0 );
    wAccessKey.setLayoutData( fdAccessKey );
    lastControl = wAccessKey;

    // Secret key
    Label wlSecretKey = new Label( shell, SWT.RIGHT );
    wlSecretKey.setText( Messages.getString( "S3CsvInputDialog.SecretKey.Label" ) ); //$NON-NLS-1$
    props.setLook( wlSecretKey );
    FormData fdlSecretKey = new FormData();
    fdlSecretKey.top = new FormAttachment( lastControl, margin );
    fdlSecretKey.left = new FormAttachment( 0, 0 );
    fdlSecretKey.right = new FormAttachment( middle, -margin );
    wlSecretKey.setLayoutData( fdlSecretKey );

    wSecretKey = new PasswordTextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );

    props.setLook( wSecretKey );
    wSecretKey.addModifyListener( lsMod );
    FormData fdSecretKey = new FormData();
    fdSecretKey.top = new FormAttachment( lastControl, margin );
    fdSecretKey.left = new FormAttachment( middle, 0 );
    fdSecretKey.right = new FormAttachment( 100, 0 );
    wSecretKey.setLayoutData( fdSecretKey );
    lastControl = wSecretKey;

    // Bucket name
    Label wlBucket = new Label( shell, SWT.RIGHT );
    wlBucket.setText( Messages.getString( "S3CsvInputDialog.Bucket.Label" ) ); //$NON-NLS-1$
    props.setLook( wlBucket );
    FormData fdlBucket = new FormData();
    fdlBucket.top = new FormAttachment( lastControl, margin );
    fdlBucket.left = new FormAttachment( 0, 0 );
    fdlBucket.right = new FormAttachment( middle, -margin );
    wlBucket.setLayoutData( fdlBucket );
    wbBucket = new Button( shell, SWT.PUSH | SWT.CENTER );
    props.setLook( wbBucket );
    wbBucket.setText( Messages.getString( "S3CsvInputDialog.Bucket.Button" ) );
    FormData fdbBucket = new FormData();
    fdbBucket.top = new FormAttachment( lastControl, margin );
    fdbBucket.right = new FormAttachment( 100, 0 );
    wbBucket.setLayoutData( fdbBucket );
    wBucket = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wBucket );
    wBucket.addModifyListener( lsMod );
    FormData fdBucket = new FormData();
    fdBucket.top = new FormAttachment( lastControl, margin );
    fdBucket.left = new FormAttachment( middle, 0 );
    fdBucket.right = new FormAttachment( wbBucket, -margin );
    wBucket.setLayoutData( fdBucket );
    lastControl = wBucket;

    // See if the step receives input.  If so, we don't ask for the filename, but for the filename field.
    //
    isReceivingInput = transMeta.findNrPrevSteps( stepMeta ) > 0;
    if ( isReceivingInput ) {

      RowMetaInterface previousFields;
      try {
        previousFields = transMeta.getPrevStepFields( stepMeta );
      } catch ( KettleStepException e ) {
        new ErrorDialog( shell, Messages.getString( "S3CsvInputDialog.ErrorDialog.UnableToGetInputFields.Title" ), Messages.getString( "S3CsvInputDialog.ErrorDialog.UnableToGetInputFields.Message" ), e );
        previousFields = new RowMeta();
      }

      // The filename field ...
      //
      Label wlFilename = new Label( shell, SWT.RIGHT );
      wlFilename.setText( Messages.getString( "S3CsvInputDialog.FilenameField.Label" ) ); //$NON-NLS-1$
      props.setLook( wlFilename );
      FormData fdlFilename = new FormData();
      fdlFilename.top = new FormAttachment( lastControl, margin );
      fdlFilename.left = new FormAttachment( 0, 0 );
      fdlFilename.right = new FormAttachment( middle, -margin );
      wlFilename.setLayoutData( fdlFilename );
      wFilenameField = new CCombo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
      wFilenameField.setItems( previousFields.getFieldNames() );
      props.setLook( wFilenameField );
      wFilenameField.addModifyListener( lsMod );
      FormData fdFilename = new FormData();
      fdFilename.top = new FormAttachment( lastControl, margin );
      fdFilename.left = new FormAttachment( middle, 0 );
      fdFilename.right = new FormAttachment( 100, 0 );
      wFilenameField.setLayoutData( fdFilename );
      lastControl = wFilenameField;

      // Checkbox to include the filename in the output...
      //
      Label wlIncludeFilename = new Label( shell, SWT.RIGHT );
      wlIncludeFilename.setText( Messages.getString( "S3CsvInputDialog.IncludeFilenameField.Label" ) ); //$NON-NLS-1$
      props.setLook( wlIncludeFilename );
      FormData fdlIncludeFilename = new FormData();
      fdlIncludeFilename.top = new FormAttachment( lastControl, margin );
      fdlIncludeFilename.left = new FormAttachment( 0, 0 );
      fdlIncludeFilename.right = new FormAttachment( middle, -margin );
      wlIncludeFilename.setLayoutData( fdlIncludeFilename );
      wIncludeFilename = new Button( shell, SWT.CHECK );
      props.setLook( wIncludeFilename );
      wFilenameField.addModifyListener( lsMod );
      FormData fdIncludeFilename = new FormData();
      fdIncludeFilename.top = new FormAttachment( lastControl, margin );
      fdIncludeFilename.left = new FormAttachment( middle, 0 );
      fdIncludeFilename.right = new FormAttachment( 100, 0 );
      wIncludeFilename.setLayoutData( fdIncludeFilename );
      lastControl = wIncludeFilename;
    } else {

      // Filename...
      //
      // The filename browse button
      //
      wbbFilename = new Button( shell, SWT.PUSH | SWT.CENTER );
      props.setLook( wbbFilename );
      wbbFilename.setText( Messages.getString( "System.Button.Browse" ) );
      wbbFilename.setToolTipText( Messages.getString( "System.Tooltip.BrowseForFileOrDirAndAdd" ) );
      FormData fdbFilename = new FormData();
      fdbFilename.top = new FormAttachment( lastControl, margin );
      fdbFilename.right = new FormAttachment( 100, 0 );
      wbbFilename.setLayoutData( fdbFilename );

      // The field itself...
      //
      Label wlFilename = new Label( shell, SWT.RIGHT );
      wlFilename.setText( Messages.getString( "S3CsvInputDialog.Filename.Label" ) ); //$NON-NLS-1$
      props.setLook( wlFilename );
      FormData fdlFilename = new FormData();
      fdlFilename.top = new FormAttachment( lastControl, margin );
      fdlFilename.left = new FormAttachment( 0, 0 );
      fdlFilename.right = new FormAttachment( middle, -margin );
      wlFilename.setLayoutData( fdlFilename );
      wFilename = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
      props.setLook( wFilename );
      wFilename.addModifyListener( lsMod );
      FormData fdFilename = new FormData();
      fdFilename.top = new FormAttachment( lastControl, margin );
      fdFilename.left = new FormAttachment( middle, 0 );
      fdFilename.right = new FormAttachment( wbbFilename, -margin );
      wFilename.setLayoutData( fdFilename );
      lastControl = wFilename;
    }

    // delimiter
    Label wlDelimiter = new Label( shell, SWT.RIGHT );
    wlDelimiter.setText( Messages.getString( "S3CsvInputDialog.Delimiter.Label" ) ); //$NON-NLS-1$
    props.setLook( wlDelimiter );
    FormData fdlDelimiter = new FormData();
    fdlDelimiter.top = new FormAttachment( lastControl, margin );
    fdlDelimiter.left = new FormAttachment( 0, 0 );
    fdlDelimiter.right = new FormAttachment( middle, -margin );
    wlDelimiter.setLayoutData( fdlDelimiter );
    wbDelimiter = new Button( shell, SWT.PUSH | SWT.CENTER );
    props.setLook( wbDelimiter );
    wbDelimiter.setText( Messages.getString( "S3CsvInputDialog.Delimiter.Button" ) );
    FormData fdbDelimiter = new FormData();
    fdbDelimiter.top = new FormAttachment( lastControl, margin );
    fdbDelimiter.right = new FormAttachment( 100, 0 );
    wbDelimiter.setLayoutData( fdbDelimiter );
    wDelimiter = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wDelimiter );
    wDelimiter.addModifyListener( lsMod );
    FormData fdDelimiter = new FormData();
    fdDelimiter.top = new FormAttachment( lastControl, margin );
    fdDelimiter.left = new FormAttachment( middle, 0 );
    fdDelimiter.right = new FormAttachment( wbDelimiter, -margin );
    wDelimiter.setLayoutData( fdDelimiter );
    lastControl = wDelimiter;

    // enclosure
    Label wlEnclosure = new Label( shell, SWT.RIGHT );
    wlEnclosure.setText( Messages.getString( "S3CsvInputDialog.Enclosure.Label" ) ); //$NON-NLS-1$
    props.setLook( wlEnclosure );
    FormData fdlEnclosure = new FormData();
    fdlEnclosure.top = new FormAttachment( lastControl, margin );
    fdlEnclosure.left = new FormAttachment( 0, 0 );
    fdlEnclosure.right = new FormAttachment( middle, -margin );
    wlEnclosure.setLayoutData( fdlEnclosure );
    wEnclosure = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wEnclosure );
    wEnclosure.addModifyListener( lsMod );
    FormData fdEnclosure = new FormData();
    fdEnclosure.top = new FormAttachment( lastControl, margin );
    fdEnclosure.left = new FormAttachment( middle, 0 );
    fdEnclosure.right = new FormAttachment( 100, 0 );
    wEnclosure.setLayoutData( fdEnclosure );
    lastControl = wEnclosure;

    // Max line size
    //
    Label wlMaxLineSize = new Label( shell, SWT.RIGHT );
    wlMaxLineSize.setText( Messages.getString( "S3CsvInputDialog.MaxLineSize.Label" ) ); //$NON-NLS-1$
    props.setLook( wlMaxLineSize );
    FormData fdlMaxLineSize = new FormData();
    fdlMaxLineSize.top = new FormAttachment( lastControl, margin );
    fdlMaxLineSize.left = new FormAttachment( 0, 0 );
    fdlMaxLineSize.right = new FormAttachment( middle, -margin );
    wlMaxLineSize.setLayoutData( fdlMaxLineSize );
    wMaxLineSize = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wMaxLineSize );
    wMaxLineSize.addModifyListener( lsMod );
    FormData fdMaxLineSize = new FormData();
    fdMaxLineSize.top = new FormAttachment( lastControl, margin );
    fdMaxLineSize.left = new FormAttachment( middle, 0 );
    fdMaxLineSize.right = new FormAttachment( 100, 0 );
    wMaxLineSize.setLayoutData( fdMaxLineSize );
    lastControl = wMaxLineSize;

    // performingLazyConversion?
    //
    Label wlLazyConversion = new Label( shell, SWT.RIGHT );
    wlLazyConversion.setText( Messages.getString( "S3CsvInputDialog.LazyConversion.Label" ) ); //$NON-NLS-1$
    props.setLook( wlLazyConversion );
    FormData fdlLazyConversion = new FormData();
    fdlLazyConversion.top = new FormAttachment( lastControl, margin );
    fdlLazyConversion.left = new FormAttachment( 0, 0 );
    fdlLazyConversion.right = new FormAttachment( middle, -margin );
    wlLazyConversion.setLayoutData( fdlLazyConversion );
    wLazyConversion = new Button( shell, SWT.CHECK );
    props.setLook( wLazyConversion );
    FormData fdLazyConversion = new FormData();
    fdLazyConversion.top = new FormAttachment( lastControl, margin );
    fdLazyConversion.left = new FormAttachment( middle, 0 );
    fdLazyConversion.right = new FormAttachment( 100, 0 );
    wLazyConversion.setLayoutData( fdLazyConversion );
    lastControl = wLazyConversion;

    // header row?
    //
    Label wlHeaderPresent = new Label( shell, SWT.RIGHT );
    wlHeaderPresent.setText( Messages.getString( "S3CsvInputDialog.HeaderPresent.Label" ) ); //$NON-NLS-1$
    props.setLook( wlHeaderPresent );
    FormData fdlHeaderPresent = new FormData();
    fdlHeaderPresent.top = new FormAttachment( lastControl, margin );
    fdlHeaderPresent.left = new FormAttachment( 0, 0 );
    fdlHeaderPresent.right = new FormAttachment( middle, -margin );
    wlHeaderPresent.setLayoutData( fdlHeaderPresent );
    wHeaderPresent = new Button( shell, SWT.CHECK );
    props.setLook( wHeaderPresent );
    FormData fdHeaderPresent = new FormData();
    fdHeaderPresent.top = new FormAttachment( lastControl, margin );
    fdHeaderPresent.left = new FormAttachment( middle, 0 );
    fdHeaderPresent.right = new FormAttachment( 100, 0 );
    wHeaderPresent.setLayoutData( fdHeaderPresent );
    lastControl = wHeaderPresent;

        // The field itself...
        //
    Label wlRowNumField = new Label( shell, SWT.RIGHT );
    wlRowNumField.setText( Messages.getString( "S3CsvInputDialog.RowNumField.Label" ) ); //$NON-NLS-1$
    props.setLook( wlRowNumField );
    FormData fdlRowNumField = new FormData();
    fdlRowNumField.top = new FormAttachment( lastControl, margin );
    fdlRowNumField.left = new FormAttachment( 0, 0 );
    fdlRowNumField.right = new FormAttachment( middle, -margin );
    wlRowNumField.setLayoutData( fdlRowNumField );
    wRowNumField = new TextVar( transMeta, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER );
    props.setLook( wRowNumField );
    wRowNumField.addModifyListener( lsMod );
    FormData fdRowNumField = new FormData();
    fdRowNumField.top = new FormAttachment( lastControl, margin );
    fdRowNumField.left = new FormAttachment( middle, 0 );
    fdRowNumField.right = new FormAttachment( 100, 0 );
    wRowNumField.setLayoutData( fdRowNumField );
    lastControl = wRowNumField;

    // running in parallel?
    //
    Label wlRunningInParallel = new Label( shell, SWT.RIGHT );
    wlRunningInParallel.setText( Messages.getString( "S3CsvInputDialog.RunningInParallel.Label" ) ); //$NON-NLS-1$
    props.setLook( wlRunningInParallel );
    FormData fdlRunningInParallel = new FormData();
    fdlRunningInParallel.top = new FormAttachment( lastControl, margin );
    fdlRunningInParallel.left = new FormAttachment( 0, 0 );
    fdlRunningInParallel.right = new FormAttachment( middle, -margin );
    wlRunningInParallel.setLayoutData( fdlRunningInParallel );
    wRunningInParallel = new Button( shell, SWT.CHECK );
    props.setLook( wRunningInParallel );
    FormData fdRunningInParallel = new FormData();
    fdRunningInParallel.top = new FormAttachment( lastControl, margin );
    fdRunningInParallel.left = new FormAttachment( middle, 0 );
    wRunningInParallel.setLayoutData( fdRunningInParallel );
    lastControl = wRunningInParallel;

    // Some buttons first, so that the dialog scales nicely...
    //
    wOK = new Button( shell, SWT.PUSH );
    wOK.setText( Messages.getString( "System.Button.OK" ) ); //$NON-NLS-1$
    wCancel = new Button( shell, SWT.PUSH );
    wCancel.setText( Messages.getString( "System.Button.Cancel" ) ); //$NON-NLS-1$
    wPreview = new Button( shell, SWT.PUSH );
    wPreview.setText( Messages.getString( "System.Button.Preview" ) ); //$NON-NLS-1$
    wPreview.setEnabled( !isReceivingInput );
    wGet = new Button( shell, SWT.PUSH );
    wGet.setText( Messages.getString( "System.Button.GetFields" ) ); //$NON-NLS-1$
    wGet.setEnabled( !isReceivingInput );

    setButtonPositions( new Button[] { wOK, wCancel, wPreview, wGet, }, margin, null );


    // Fields
    ColumnInfo[] colinf = new ColumnInfo[] {
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.NameColumn.Column" ),       ColumnInfo.COLUMN_TYPE_TEXT,    false ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.TypeColumn.Column" ),       ColumnInfo.COLUMN_TYPE_CCOMBO,  ValueMeta.getTypes(), true ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.FormatColumn.Column" ),     ColumnInfo.COLUMN_TYPE_CCOMBO,  Const.getConversionFormats() ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.LengthColumn.Column" ),     ColumnInfo.COLUMN_TYPE_TEXT,    false ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.PrecisionColumn.Column" ),  ColumnInfo.COLUMN_TYPE_TEXT,    false ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.CurrencyColumn.Column" ),   ColumnInfo.COLUMN_TYPE_TEXT,    false ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.DecimalColumn.Column" ),    ColumnInfo.COLUMN_TYPE_TEXT,    false ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.GroupColumn.Column" ),      ColumnInfo.COLUMN_TYPE_TEXT,    false ),
      new ColumnInfo( Messages.getString( "S3CsvInputDialog.TrimTypeColumn.Column" ),   ColumnInfo.COLUMN_TYPE_CCOMBO,  ValueMeta.trimTypeDesc ),
    };

    colinf[2].setComboValuesSelectionListener( new ComboValuesSelectionListener() {
      @Override
      public String[] getComboValues( TableItem tableItem, int rowNr, int colNr ) {
        String[] comboValues = new String[] { };
        int type = ValueMeta.getType( tableItem.getText( colNr - 1 ) );
        switch ( type ) {
          case ValueMetaInterface.TYPE_DATE:
            comboValues = Const.getDateFormats();
            break;
          case ValueMetaInterface.TYPE_INTEGER:
          case ValueMetaInterface.TYPE_BIGNUMBER:
          case ValueMetaInterface.TYPE_NUMBER:
            comboValues = Const.getNumberFormats();
            break;
          default:
            break;
        }
        return comboValues;
      }

    } );


    wFields = new TableView( transMeta, shell, SWT.FULL_SELECTION | SWT.MULTI, colinf, 1, lsMod, props );

    FormData fdFields = new FormData();
    fdFields.top = new FormAttachment( lastControl, margin * 2 );
    fdFields.bottom = new FormAttachment( wOK, -margin * 2 );
    fdFields.left = new FormAttachment( 0, 0 );
    fdFields.right = new FormAttachment( 100, 0 );
    wFields.setLayoutData( fdFields );

    // Add listeners
    lsCancel = new Listener() {
      @Override
      public void handleEvent( Event e ) {
        cancel();
      }
    };
    lsOK = new Listener() {
      @Override
      public void handleEvent( Event e ) {
        ok();
      }
    };
    lsPreview = new Listener() {
      @Override
      public void handleEvent( Event e ) {
        preview();
      }
    };
    lsGet = new Listener() {
      @Override
      public void handleEvent( Event e ) {
        getCSV();
      }
    };

    wCancel.addListener( SWT.Selection, lsCancel );
    wOK.addListener( SWT.Selection, lsOK );
    wPreview.addListener( SWT.Selection, lsPreview );
    wGet.addListener( SWT.Selection, lsGet );

    lsDef = new SelectionAdapter() {
      @Override
      public void widgetDefaultSelected( SelectionEvent e ) {
        ok();
      }
    };

    wStepname.addSelectionListener( lsDef );
    if ( wFilename != null ) {
      wFilename.addSelectionListener( lsDef );
    }
    if ( wFilenameField != null ) {
      wFilenameField.addSelectionListener( lsDef );
    }
    wDelimiter.addSelectionListener( lsDef );
    wEnclosure.addSelectionListener( lsDef );
    wMaxLineSize.addSelectionListener( lsDef );
    wRowNumField.addSelectionListener( lsDef );

    // Allow the insertion of tabs as separator...
    wbDelimiter.addSelectionListener( new SelectionAdapter() {
      @Override
      public void widgetSelected( SelectionEvent se ) {
        Text t = wDelimiter.getTextWidget();
        if ( t != null ) {
          t.insert( "\t" );
        }
      }
    } );

    wbBucket.addSelectionListener( new SelectionAdapter() {
      @Override
      public void widgetSelected( SelectionEvent event ) {
        // List the buckets...
        //
        try {
          S3CsvInputMeta meta = new S3CsvInputMeta();
          getInfo( meta );
          S3Service service = meta.getS3Service( transMeta );
          S3Bucket[] buckets = service.listAllBuckets();
          String[] bucketNames = new String[buckets.length];
          for ( int i = 0; i < buckets.length; i++ ) {
            bucketNames[i] = buckets[i].getName();
          }

          EnterSelectionDialog dialog = new EnterSelectionDialog( shell, bucketNames,
              Messages.getString( "S3CsvInputDialog.Exception.SelectBucket.Title" ),
              Messages.getString( "S3CsvInputDialog.Exception.SelectBucket.Message" ) );
          dialog.setMulti( false );
          String bucketname = dialog.open();
          if ( bucketname != null ) {
            wBucket.setText( bucketname );
          }
        } catch ( Exception e ) {
          new ErrorDialog( shell,
              Messages.getString( "S3CsvInputDialog.Exception.UnableToGetBuckets.Title" ),
              Messages.getString( "S3CsvInputDialog.Exception.UnableToGetBuckets.Message" ), e );
        }
      }
    } );


    if ( wbbFilename != null ) {
      // Listen to the browse button next to the file name
      wbbFilename.addSelectionListener( new SelectionAdapter() {
        @Override
        public void widgetSelected( SelectionEvent event ) {
          try {
            S3CsvInputMeta meta = new S3CsvInputMeta();
            getInfo( meta );
            S3Service service = meta.getS3Service( transMeta );

            S3Bucket[] buckets = service.listAllBuckets();
            S3Bucket bucket = null;

            for ( S3Bucket compare : buckets ) {
              if ( compare.getName().equals( meta.getBucket() ) ) {
                bucket = compare;
              }
            }

            if ( bucket == null ) {
              throw new Exception( Messages.getString( "S3CsvInputDialog.Exception.UnableToFindBucket.Message", meta.getBucket() ) );
            }

            S3Object[] objects = service.listObjects( bucket );
            String[] objectnames = new String[objects.length];
            for ( int i = 0; i < objects.length; i++ ) {
              objectnames[i] = objects[i].getKey();
            }

            EnterSelectionDialog dialog = new EnterSelectionDialog( shell, objectnames,
              Messages.getString( "S3CsvInputDialog.Exception.SelectObject.Title" ),
              Messages.getString( "S3CsvInputDialog.Exception.SelectObject.Message" ) );
            dialog.setMulti( false );
            if ( !Utils.isEmpty( wFilename.getText() ) ) {
              int index = Const.indexOfString( wFilename.getText(), objectnames );
              if ( index >= 0 ) {
                dialog.setSelectedNrs( new int[] { index, } );
              }
            }
            String objectname = dialog.open();
            if ( objectname != null ) {
              wFilename.setText( objectname );
            }
          } catch ( Exception e ) {
            new ErrorDialog( shell,
              Messages.getString( "S3CsvInputDialog.Exception.UnableToGetFiles.Title" ),
              Messages.getString( "S3CsvInputDialog.Exception.UnableToGetFiles.Message" ), e );
          }
        }
      } );
    }


    // Detect X or ALT-F4 or something that kills this window...
    shell.addShellListener( new ShellAdapter() {
      @Override
      public void shellClosed( ShellEvent e ) {
        cancel();
      }
    } );


    // Set the shell size, based upon previous time...
    setSize();

    getData();
    inputMeta.setChanged( changed );

    shell.open();
    while ( !shell.isDisposed() ) {
      if ( !display.readAndDispatch() ) {
        display.sleep();
      }
    }
    return stepname;
  }

  public void getData() {
    getData( inputMeta );
  }
  /**
   * Copy information from the meta-data input to the dialog fields.
   */
  public void getData( S3CsvInputMeta inputMeta ) {
    wStepname.setText( stepname );
    wAccessKey.setText( Const.NVL( inputMeta.getAwsAccessKey(), "" ) );
    wSecretKey.setText( Const.NVL( inputMeta.getAwsSecretKey(), "" ) );
    wBucket.setText( Const.NVL( inputMeta.getBucket(), "" ) );

    if ( isReceivingInput ) {
      wFilenameField.setText( Const.NVL( inputMeta.getFilenameField(), "" ) );
      wIncludeFilename.setSelection( inputMeta.isIncludingFilename() );
    } else {
      wFilename.setText( Const.NVL( inputMeta.getFilename(), "" ) );
    }
    wDelimiter.setText( Const.NVL( inputMeta.getDelimiter(), "" ) );
    wEnclosure.setText( Const.NVL( inputMeta.getEnclosure(), "" ) );
    wMaxLineSize.setText( Const.NVL( inputMeta.getMaxLineSize(), "" ) );
    wLazyConversion.setSelection( inputMeta.isLazyConversionActive() );
    wHeaderPresent.setSelection( inputMeta.isHeaderPresent() );
    wRunningInParallel.setSelection( inputMeta.isRunningInParallel() );
    wRowNumField.setText( Const.NVL( inputMeta.getRowNumField(), "" ) );

    for ( int i = 0; i < inputMeta.getInputFields().length; i++ ) {
      TextFileInputField field = inputMeta.getInputFields()[i];

      TableItem item = new TableItem( wFields.table, SWT.NONE );
      int colnr = 1;
      item.setText( colnr++, Const.NVL( field.getName(), "" ) );
      item.setText( colnr++, ValueMeta.getTypeDesc( field.getType() ) );
      item.setText( colnr++, Const.NVL( field.getFormat(), "" ) );
      item.setText( colnr++, field.getLength() >= 0 ? Integer.toString( field.getLength() ) : "" );
      item.setText( colnr++, field.getPrecision() >= 0 ? Integer.toString( field.getPrecision() ) : "" );
      item.setText( colnr++, Const.NVL( field.getCurrencySymbol(), "" ) );
      item.setText( colnr++, Const.NVL( field.getDecimalSymbol(), "" ) );
      item.setText( colnr++, Const.NVL( field.getGroupSymbol(), "" ) );
      item.setText( colnr++, Const.NVL( field.getTrimTypeDesc(), "" ) );
    }
    wFields.removeEmptyRows();
    wFields.setRowNums();
    wFields.optWidth( true );

    wStepname.selectAll();
  }

  private void cancel() {
    stepname = null;
    inputMeta.setChanged( changed );
    dispose();
  }

  private void getInfo( S3CsvInputMeta inputMeta ) {

    inputMeta.setAwsAccessKey( wAccessKey.getText() );
    inputMeta.setAwsSecretKey( wSecretKey.getText() );
    inputMeta.setBucket( wBucket.getText() );

    if ( isReceivingInput ) {
      inputMeta.setFilenameField( wFilenameField.getText() );
      inputMeta.setIncludingFilename( wIncludeFilename.getSelection() );
    } else {
      inputMeta.setFilename( wFilename.getText() );
    }

    inputMeta.setDelimiter( wDelimiter.getText() );
    inputMeta.setEnclosure( wEnclosure.getText() );
    inputMeta.setMaxLineSize( wMaxLineSize.getText() );
    inputMeta.setLazyConversionActive( wLazyConversion.getSelection() );
    inputMeta.setHeaderPresent( wHeaderPresent.getSelection() );
    inputMeta.setRowNumField( wRowNumField.getText() );
    inputMeta.setRunningInParallel( wRunningInParallel.getSelection() );

    int nrNonEmptyFields = wFields.nrNonEmpty();
    inputMeta.allocate( nrNonEmptyFields );

    for ( int i = 0; i < nrNonEmptyFields; i++ ) {
      TableItem item = wFields.getNonEmpty( i );
      inputMeta.getInputFields()[i] = new TextFileInputField();

      int colnr = 1;
      inputMeta.getInputFields()[i].setName( item.getText( colnr++ ) );
      inputMeta.getInputFields()[i].setType( ValueMeta.getType( item.getText( colnr++ ) ) );
      inputMeta.getInputFields()[i].setFormat( item.getText( colnr++ ) );
      inputMeta.getInputFields()[i].setLength( Const.toInt( item.getText( colnr++ ), -1 ) );
      inputMeta.getInputFields()[i].setPrecision( Const.toInt( item.getText( colnr++ ), -1 ) );
      inputMeta.getInputFields()[i].setCurrencySymbol( item.getText( colnr++ ) );
      inputMeta.getInputFields()[i].setDecimalSymbol( item.getText( colnr++ ) );
      inputMeta.getInputFields()[i].setGroupSymbol( item.getText( colnr++ ) );
      inputMeta.getInputFields()[i].setTrimType( ValueMeta.getTrimTypeByDesc( item.getText( colnr++ ) ) );
    }
    wFields.removeEmptyRows();
    wFields.setRowNums();
    wFields.optWidth( true );

    inputMeta.setChanged();
  }

  private void ok() {
    if ( Utils.isEmpty( wStepname.getText() ) ) {
      return;
    }

    getInfo( inputMeta );
    stepname = wStepname.getText();
    dispose();
  }

  // Get the data layout
  private void getCSV() {
    InputStream inputStream = null;
    try {
      S3CsvInputMeta meta = new S3CsvInputMeta();
      getInfo( meta );

      String filename = transMeta.environmentSubstitute( meta.getFilename() );
      String bucketname = transMeta.environmentSubstitute( meta.getBucket() );
      int maxLineSize = Const.toInt( transMeta.environmentSubstitute( meta.getMaxLineSize() ), 2000 );

      wFields.table.removeAll();

      S3Service s3Service = meta.getS3Service( transMeta );
      S3Bucket[] buckets = s3Service.listAllBuckets();
      S3Bucket s3bucket = null;
      for ( S3Bucket bucket : buckets ) {
        if ( bucket.getName().equals( bucketname ) ) {
          s3bucket = bucket;
        }
      }

      if ( s3bucket == null ) {
        throw new Exception( "Unable to find specified bucket : [" + bucketname + "]" ); // TODO i18n
      }

      // Now we can continue reading the rows of data and we can guess the
      // Sample a few lines to determine the correct type of the fields...
      //
      String shellText = Messages.getString( "S3CsvInputDialog.LinesToSample.DialogTitle" );
      String lineText = Messages.getString( "S3CsvInputDialog.LinesToSample.DialogMessage" );
      EnterNumberDialog end = new EnterNumberDialog( shell, 100, shellText, lineText );
      int samples = end.open();
      if ( samples < 0 ) {
        return;
      }

      // Only get the first lines, not the complete file
      //
      S3Object object = s3Service.getObject( s3bucket, filename, null, null, null, null, 0L, (long) samples * (long) maxLineSize );

      // Grab an input stream to the data...
      //
      inputStream = object.getDataInputStream();

      InputStreamReader reader = new InputStreamReader( inputStream );

      // Read a line of data to determine the number of rows...
      //
      String line = TextFileInput.getLine( log, reader, TextFileInputMeta.FILE_FORMAT_MIXED, new StringBuilder( 1000 ) );

      // Split the string, header or data into parts...
      //
      String[] fieldNames = Const.splitString( line, meta.getDelimiter() );

      if ( !meta.isHeaderPresent() ) {
        // Don't use field names from the header...
        // Generate field names F1 ... F10
        //
        DecimalFormat df = new DecimalFormat( "000" ); // $NON-NLS-1$
        for ( int i = 0; i < fieldNames.length; i++ ) {
          fieldNames[i] = "Field_" + df.format( i ); // $NON-NLS-1$
        }
      } else {
        if ( !Utils.isEmpty( meta.getEnclosure() ) ) {
          for ( int i = 0; i < fieldNames.length; i++ ) {
            if ( fieldNames[i].startsWith( meta.getEnclosure() ) && fieldNames[i].endsWith( meta.getEnclosure() ) && fieldNames[i].length() > 1 ) {
              fieldNames[i] = fieldNames[i].substring( 1, fieldNames[i].length() - 1 );
            }
          }
        }
      }

      // Trim the names to make sure...
      //
      for ( int i = 0; i < fieldNames.length; i++ ) {
        fieldNames[i] = Const.trim( fieldNames[i] );
      }

      // Update the GUI
      //
      for ( int i = 0; i < fieldNames.length; i++ ) {
        TableItem item = new TableItem( wFields.table, SWT.NONE );
        item.setText( 1, fieldNames[i] );
        item.setText( 2, ValueMeta.getTypeDesc( ValueMetaInterface.TYPE_STRING ) );
      }
      wFields.removeEmptyRows();
      wFields.setRowNums();
      wFields.optWidth( true );

      getInfo( meta );

      TextFileCSVImportProgressDialog pd = new TextFileCSVImportProgressDialog( shell, meta, transMeta, reader, samples, true );
      String message = pd.open();
      if ( message != null ) {
        wFields.removeAll();

        // OK, what's the result of our search?
        getData( meta );
        wFields.removeEmptyRows();
        wFields.setRowNums();
        wFields.optWidth( true );

        EnterTextDialog etd = new EnterTextDialog( shell, Messages.getString( "S3CsvInputDialog.ScanResults.DialogTitle" ), Messages.getString( "S3CsvInputDialog.ScanResults.DialogMessage" ), message, true );
        etd.setReadOnly();
        etd.open();
      }
    } catch ( IOException e ) {
      new ErrorDialog( shell, Messages.getString( "S3CsvInputDialog.IOError.DialogTitle" ), Messages.getString( "S3CsvInputDialog.IOError.DialogMessage" ), e );
    } catch ( Exception e ) {
      new ErrorDialog( shell, Messages.getString( "System.Dialog.Error.Title" ), Messages.getString( "S3CsvInputDialog.ErrorGettingFileDesc.DialogMessage" ), e );
    } finally {
      try {
        if ( inputStream != null ) {
          inputStream.close();
        }
      } catch ( Exception e ) {
        log.logError( stepname, "Error closing s3 data input stream", e );
      }
    }
  }

  // Preview the data
  private void preview() {
    // Create the XML input step
    S3CsvInputMeta oneMeta = new S3CsvInputMeta();
    getInfo( oneMeta );

    TransMeta previewMeta = TransPreviewFactory.generatePreviewTransformation( transMeta, oneMeta, wStepname.getText() );

    EnterNumberDialog numberDialog = new EnterNumberDialog( shell, props.getDefaultPreviewSize(), Messages.getString( "S3CsvInputDialog.PreviewSize.DialogTitle" ), Messages.getString( "S3CsvInputDialog.PreviewSize.DialogMessage" ) );
    int previewSize = numberDialog.open();
    if ( previewSize > 0 ) {
      TransPreviewProgressDialog progressDialog = new TransPreviewProgressDialog( shell, previewMeta, new String[] { wStepname.getText() }, new int[] { previewSize } );
      progressDialog.open();

      Trans trans = progressDialog.getTrans();
      String loggingText = progressDialog.getLoggingText();

      if ( !progressDialog.isCancelled() ) {
        if ( trans.getResult() != null && trans.getResult().getNrErrors() > 0 ) {
          EnterTextDialog etd = new EnterTextDialog( shell, Messages.getString( "System.Dialog.PreviewError.Title" ),
            Messages.getString( "System.Dialog.PreviewError.Message" ), loggingText, true );
          etd.setReadOnly();
          etd.open();
        }
      }

      PreviewRowsDialog prd = new PreviewRowsDialog( shell, transMeta, SWT.NONE, wStepname.getText(), progressDialog.getPreviewRowsMeta( wStepname.getText() ), progressDialog.getPreviewRows( wStepname.getText() ), loggingText );
      prd.open();
    }
  }
}