HTTPPOSTIT.java

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2002-2017 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.trans.steps.httppost;

import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;


import java.io.IOException;
import java.io.InputStreamReader;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.URLDecoder;
import java.util.concurrent.atomic.AtomicBoolean;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HostConfiguration;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.PostMethod;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.InjectMocks;
import org.mockito.Mockito;
import org.pentaho.di.core.KettleClientEnvironment;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.logging.LoggingObjectInterface;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.value.ValueMetaInteger;
import org.pentaho.di.core.row.value.ValueMetaString;
import org.pentaho.di.core.util.Assert;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.steps.mock.StepMockHelper;

import com.google.common.io.ByteStreams;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;

/**
 * User: Dzmitry Stsiapanau Date: 12/2/13 Time: 4:35 PM
 */
public class HTTPPOSTIT {
  class HTTPPOSTHandler extends HTTPPOST {

    Object[] row = new Object[] { "anyData" };
    Object[] outputRow;
    boolean  override;

    public HTTPPOSTHandler( StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
                      Trans trans, boolean override ) {
      super( stepMeta, stepDataInterface, copyNr, transMeta, trans );
      this.override = override;
    }


    /**
     * In case of getRow, we receive data from previous steps through the input rowset. In case we split the stream, we
     * have to copy the data to the alternate splits: rowsets 1 through n.
     */
    @Override
    public Object[] getRow() throws KettleException {
      return row;
    }

    /**
     * putRow is used to copy a row, to the alternate rowset(s) This should get priority over everything else!
     * (synchronized) If distribute is true, a row is copied only once to the output rowsets, otherwise copies are sent
     * to each rowset!
     *
     * @param row
     *        The row to put to the destination rowset(s).
     * @throws org.pentaho.di.core.exception.KettleStepException
     *
     */
    @Override
    public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException {
      outputRow = row;
    }

    public Object[] getOutputRow() {
      return outputRow;
    }


    @Override
    protected int requestStatusCode( PostMethod post, HostConfiguration hostConfiguration, HttpClient httpPostClient )
      throws IOException {
      if ( override ) {
        return 402;
      } else {
        return super.requestStatusCode( post, hostConfiguration, httpPostClient );
      }

    }

    @Override
    protected InputStreamReader openStream( String encoding, PostMethod post ) throws Exception {
      if ( override ) {
        InputStreamReader mockInputStreamReader = Mockito.mock( InputStreamReader.class );
        when( mockInputStreamReader.read() ).thenReturn( -1 );
        return mockInputStreamReader;
      } else {
        return super.openStream( encoding, post );
      }
    }

    @Override
    protected Header[] searchForHeaders( PostMethod post ) {
      Header[] headers = { new Header( "host", host ) };
      if ( override ) {
        return headers;
      } else {
        return super.searchForHeaders( post );
      }
    }
  }

  public static final String host = "localhost";
  public static final int port = 9998;
  public static final String HTTP_LOCALHOST_9998 = "http://localhost:9998/";

  @InjectMocks
  private StepMockHelper<HTTPPOSTMeta, HTTPPOSTData> stepMockHelper;
  private HttpServer httpServer;

  @BeforeClass
  public static void setupBeforeClass() throws KettleException {
    KettleClientEnvironment.init();
  }

  @Before
  public void setUp() throws Exception {
    stepMockHelper =
      new StepMockHelper<HTTPPOSTMeta, HTTPPOSTData>( "HTTPPOST CLIENT TEST",
      HTTPPOSTMeta.class, HTTPPOSTData.class );
    when( stepMockHelper.logChannelInterfaceFactory.create( any(), any( LoggingObjectInterface.class ) ) ).thenReturn(
      stepMockHelper.logChannelInterface );
    when( stepMockHelper.trans.isRunning() ).thenReturn( true );
    verify( stepMockHelper.trans, never() ).stopAll();
  }

  @After
  public void tearDown() throws Exception {
    httpServer.stop( 5 );

  }

  @Test
  public void test204Answer() throws Exception {
    startHttpServer( get204AnswerHandler() );
    HTTPPOSTData data = new HTTPPOSTData();
    int[] index = { 0, 1 };
    RowMeta meta = new RowMeta();
    meta.addValueMeta( new ValueMetaString( "fieldName" ) );
    meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) );
    Object[] expectedRow = new Object[] { "", 204L };
    HTTPPOST HTTPPOST = new HTTPPOSTHandler(
      stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false );
    RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
    HTTPPOST.setInputRowMeta( inputRowMeta );
    when( inputRowMeta.clone() ).thenReturn( inputRowMeta );
    when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 );
    when( stepMockHelper.processRowsStepMetaInterface.getQueryField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getResultCodeFieldName() ).thenReturn( "ResultCodeFieldName" );
    when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" );
    HTTPPOST.init( stepMockHelper.processRowsStepMetaInterface, data );
    Assert.assertTrue( HTTPPOST.processRow( stepMockHelper.processRowsStepMetaInterface, data ) );
    Object[] out = ( (HTTPPOSTHandler) HTTPPOST ).getOutputRow();
    Assert.assertTrue( meta.equals( out, expectedRow, index ) );
  }

  @Test
  public void testResponseHeader() throws Exception {
    startHttpServer( get204AnswerHandler() );
    HTTPPOSTData data = new HTTPPOSTData();
    int[] index = { 0, 1, 3 };
    RowMeta meta = new RowMeta();
    meta.addValueMeta( new ValueMetaString( "fieldName" ) );
    meta.addValueMeta( new ValueMetaInteger( "codeFieldName" ) );
    meta.addValueMeta( new ValueMetaInteger( "responseTimeFieldName" ) );
    meta.addValueMeta( new ValueMetaString( "headerFieldName" ) );
    Object[] expectedRow =
      new Object[] { "", 402L, 0L, "{\"host\":\"localhost\"}" };
    HTTPPOST HTTPPOST = new HTTPPOSTHandler(
      stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, true );
    RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
    HTTPPOST.setInputRowMeta( inputRowMeta );
    when( inputRowMeta.clone() ).thenReturn( inputRowMeta );
    when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 );
    when( stepMockHelper.processRowsStepMetaInterface.getQueryField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getResultCodeFieldName() ).thenReturn( "ResultCodeFieldName" );
    when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" );
    when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( "UTF-8" );
    when( stepMockHelper.processRowsStepMetaInterface.getResponseTimeFieldName() ).thenReturn(
      "ResponseTimeFieldName" );
    when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn(
      "ResponseHeaderFieldName" );
    HTTPPOST.init( stepMockHelper.processRowsStepMetaInterface, data );
    Assert.assertTrue( HTTPPOST.processRow( stepMockHelper.processRowsStepMetaInterface, data ) );
    Object[] out = ( (HTTPPOSTHandler) HTTPPOST ).getOutputRow();
    Assert.assertTrue( meta.equals( out, expectedRow, index ) );
  }

  @Test
  public void testDuplicateNamesInHeader() throws Exception {
    startHttpServer( getDuplicateHeadersHandler() );
    HTTPPOSTData data = new HTTPPOSTData();
    RowMeta meta = new RowMeta();
    meta.addValueMeta( new ValueMetaString( "headerFieldName" ) );
    HTTPPOST HTTPPOST = new HTTPPOSTHandler(
      stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false );
    RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
    HTTPPOST.setInputRowMeta( inputRowMeta );
    when( inputRowMeta.clone() ).thenReturn( inputRowMeta );
    when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 );
    when( stepMockHelper.processRowsStepMetaInterface.getQueryField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( "UTF-8" );
    when( stepMockHelper.processRowsStepMetaInterface.getResponseHeaderFieldName() ).thenReturn(
      "ResponseHeaderFieldName" );
    HTTPPOST.init( stepMockHelper.processRowsStepMetaInterface, data );
    Assert.assertTrue( HTTPPOST.processRow( stepMockHelper.processRowsStepMetaInterface, data ) );
    Object[] out = ( (HTTPPOSTHandler) HTTPPOST ).getOutputRow();
    Assert.assertTrue( out.length == 1 );
    JSONParser parser = new JSONParser();
    JSONObject json = (JSONObject) parser.parse( (String) out[0] );
    Object userAgent = json.get( "User-agent" );
    Assert.assertTrue( "HTTPTool/1.0".equals( userAgent ) );
    Object cookies = json.get( "Set-cookie" );
    Assert.assertTrue( cookies instanceof JSONArray );
    for ( int i = 0; i < 3; i++ ) {
      String cookie = ( (String) ( (JSONArray) cookies ).get( i ) );
      Assert.assertTrue( cookie.startsWith( "cookie" + i ) );
    }
  }

  @Test
  public void testUTF8() throws Exception {
    testServerReturnsCorrectlyEncodedParams( "test string \uD842\uDFB7 øó 測試", "UTF-8" );
  }

  @Test
  public void testUTF16() throws Exception {
    testServerReturnsCorrectlyEncodedParams( "test string \uD842\uDFB7 øó 測試", "UTF-16" );
  }

  @Test
  public void testUTF32() throws Exception {
    testServerReturnsCorrectlyEncodedParams( "test string \uD842\uDFB7 øó 測試", "UTF-32" );
  }

  public void testServerReturnsCorrectlyEncodedParams( String testString, String testCharset ) throws Exception {
    AtomicBoolean testStatus = new AtomicBoolean();
    startHttpServer( getEncodingCheckingHandler( testString, testCharset, testStatus ) );
    HTTPPOSTData data = new HTTPPOSTData();
    RowMeta meta = new RowMeta();
    meta.addValueMeta( new ValueMetaString( "fieldName" ) );
    HTTPPOSTHandler httpPost = new HTTPPOSTHandler(
      stepMockHelper.stepMeta, data, 0, stepMockHelper.transMeta, stepMockHelper.trans, false );
    RowMetaInterface inputRowMeta = mock( RowMetaInterface.class );
    httpPost.setInputRowMeta( inputRowMeta );
    httpPost.row = new Object[] { testString };
    when( inputRowMeta.clone() ).thenReturn( inputRowMeta );
    when( inputRowMeta.getString( httpPost.row, 0 ) ).thenReturn( testString );
    when( stepMockHelper.processRowsStepMetaInterface.getUrl() ).thenReturn( HTTP_LOCALHOST_9998 );
    when( stepMockHelper.processRowsStepMetaInterface.getQueryField() ).thenReturn( new String[] {} );
    when( stepMockHelper.processRowsStepMetaInterface.getArgumentField() ).thenReturn( new String[] { "testBodyField" } );
    when( stepMockHelper.processRowsStepMetaInterface.getArgumentParameter() ).thenReturn( new String[] { "testBodyParam" } );
    when( stepMockHelper.processRowsStepMetaInterface.getArgumentHeader() ).thenReturn( new boolean[] { false } );
    when( stepMockHelper.processRowsStepMetaInterface.getFieldName() ).thenReturn( "ResultFieldName" );
    when( stepMockHelper.processRowsStepMetaInterface.getEncoding() ).thenReturn( testCharset );
    httpPost.init( stepMockHelper.processRowsStepMetaInterface, data );
    Assert.assertTrue( httpPost.processRow( stepMockHelper.processRowsStepMetaInterface, data ) );
    Assert.assertTrue( testStatus.get(), "Test failed" );
  }


  private void startHttpServer( HttpHandler httpHandler ) throws IOException {
    httpServer = HttpServer.create( new InetSocketAddress( HTTPPOSTIT.host, HTTPPOSTIT.port ), 10 );
    httpServer.createContext( "/", httpHandler );
    httpServer.start();
  }

  private HttpHandler get204AnswerHandler() {
    return httpExchange -> {
      httpExchange.sendResponseHeaders( 204, 0 );
      httpExchange.close();
    };
  }

  private HttpHandler getDuplicateHeadersHandler() {
    return httpExchange -> {
      Headers headers = httpExchange.getResponseHeaders();
      headers.add( "User-agent", "HTTPTool/1.0" );
      headers.add( "Set-cookie", "cookie0=value0; Max-Age=3600" );
      headers.add( "Set-cookie", "cookie1=value1; HttpOnly" );
      headers.add( "Set-cookie", "cookie2=value2; Secure" );
      httpExchange.sendResponseHeaders( 200, 0 );
      httpExchange.close();
    };
  }

  private HttpHandler getEncodingCheckingHandler( String expectedResultString, String expectedEncoding, AtomicBoolean testStatus ) {
    return httpExchange -> {
      try {
        checkEncoding( expectedResultString, expectedEncoding, httpExchange.getRequestBody() );
        testStatus.set( true );
      } catch ( Throwable e ) {
        e.printStackTrace();
        testStatus.set( false );
      } finally {
        httpExchange.sendResponseHeaders( 200, 0 );
        httpExchange.close();
      }
    };
  }

  private void checkEncoding( String expectedResult, String encoding, InputStream inputStream ) throws Exception {
    byte[] receivedBytes = ByteStreams.toByteArray( inputStream );
    String urlEncodedString = new String( receivedBytes, "US-ASCII" );
    String finalString = URLDecoder.decode( urlEncodedString, encoding );
    expectedResult = "testBodyParam=" + expectedResult;
    assertEquals( "The final received string is not the same", expectedResult, finalString );
  }

}