FastJsonReader.java

/*! ******************************************************************************
 *
 * Pentaho Data Integration
 *
 * Copyright (C) 2016 by Pentaho : http://www.pentaho.com
 *
 *******************************************************************************
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ******************************************************************************/

package org.pentaho.di.trans.steps.jsoninput.reader;

import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;

import org.pentaho.di.core.RowSet;
import org.pentaho.di.core.SingleRowRowSet;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.logging.LogChannelInterface;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.steps.jsoninput.JsonInputField;
import org.pentaho.di.trans.steps.jsoninput.JsonInputMeta;

import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import com.jayway.jsonpath.ParseContext;
import com.jayway.jsonpath.ReadContext;
import org.pentaho.di.trans.steps.jsoninput.exception.JsonInputException;

/**
 * @author Samatar
 * @author edube
 * @author jadametz
 * @since 2015-08-18
 */
public class FastJsonReader implements IJsonReader {
  private static Class<?> PKG = JsonInputMeta.class; // for i18n purposes, needed by Translator2!!

  // as per RFC 7159, the default JSON encoding shall be UTF-8
  // see https://tools.ietf.org/html/rfc7159#section-8.1
  private static final String JSON_CHARSET = "UTF-8";

  private ReadContext jsonReadContext;
  private Configuration jsonConfiguration;

  private boolean ignoreMissingPath;

  private JsonInputField[] fields;
  private JsonPath[] paths = null;
  private LogChannelInterface log;

  private static final Option[] DEFAULT_OPTIONS =
    { Option.SUPPRESS_EXCEPTIONS, Option.ALWAYS_RETURN_LIST, Option.DEFAULT_PATH_LEAF_TO_NULL };

  protected FastJsonReader( LogChannelInterface log ) throws KettleException {
    this.ignoreMissingPath = false;
    this.jsonConfiguration = Configuration.defaultConfiguration().addOptions( DEFAULT_OPTIONS );
    this.log = log;
  }

  public FastJsonReader( JsonInputField[] fields, LogChannelInterface log ) throws KettleException {
    this( log );
    this.fields = fields;
    this.paths = compilePaths( fields );
  }

  public void setIgnoreMissingPath( boolean value ) {
    this.ignoreMissingPath = value;
  }

  private ParseContext getParseContext() {
    return JsonPath.using( jsonConfiguration );
  }

  private ReadContext getReadContext() {
    return jsonReadContext;
  }

  private static JsonPath[] compilePaths( JsonInputField[] fields ) {
    JsonPath[] paths = new JsonPath[ fields.length ];
    int i = 0;
    for ( JsonInputField field : fields ) {
      paths[ i++ ] = JsonPath.compile( field.getPath() );
    }
    return paths;
  }

  protected void readInput( InputStream is ) throws KettleException {
    jsonReadContext = getParseContext().parse( is, JSON_CHARSET );
    if ( jsonReadContext == null ) {
      throw new KettleException( BaseMessages.getString( PKG, "JsonReader.Error.ReadUrl.Null" ) );
    }
  }

  public boolean isIgnoreMissingPath() {
    return this.ignoreMissingPath;
  }

  @Override
  public void setFields( JsonInputField[] fields ) throws KettleException {
    this.fields = fields;
    this.paths = compilePaths( fields );
  }

  @Override
  public RowSet parse( InputStream in ) throws KettleException {
    readInput( in );
    List<Object[]> results = evalCombinedResult();
    if ( log.isDetailed() ) {
      log.logDetailed( BaseMessages.getString( PKG, "JsonInput.Log.NrRecords", results.size() ) );
    }
    return new TransposedRowSet( results );
  }

  private static class TransposedRowSet extends SingleRowRowSet {
    private List<Object[]> results;

    public TransposedRowSet( List<Object[]> results ) {
      super();
      this.results = results;
    }

    @Override
    public Object[] getRow() {
      if ( !results.isEmpty() ) {
        return results.remove( 0 );
      }

      return null;
    }

    @Override
    public int size() {
      return results.size();
    }

    @Override
    public boolean isDone() {
      // built at ctor
      return true;
    }

    @Override
    public void clear() {
      results.clear();
    }
  }

  private List<Object[]> evalCombinedResult() throws JsonInputException {
    int lastSize = -1;
    String prevPath = null;
    List<List<?>> inputs = new ArrayList<>( paths.length );
    int i = 0;
    for ( JsonPath path : paths ) {
      List<Object> input = getReadContext().read( path );
      if ( input.size() != lastSize && lastSize > 0 & input.size() != 0 ) {
        throw new JsonInputException( BaseMessages.getString(
            PKG, "JsonInput.Error.BadStructure", input.size(), fields[i].getPath(), prevPath, lastSize ) );
      }
      if ( ( isAllNull( input ) || input.size() == 0 ) && !isIgnoreMissingPath() ) {
        throw new JsonInputException( BaseMessages.getString( PKG, "JsonReader.Error.CanNotFindPath", fields[i].getPath() ) );
      }
      inputs.add( input );
      lastSize = input.size();
      prevPath = fields[i].getPath();
      i++;
    }

    List<Object[]> resultRows = convertInputsIntoResultRows( inputs );

    filterOutExcessRows( resultRows );

    if ( !ignoreMissingPath & paths.length != 0 ) {
      raiseExceptionIfAnyMissingPath( resultRows );
    }

    return resultRows;
  }

  private List<Object[]> convertInputsIntoResultRows( List<List<?>> inputs ) {

    List<Object[]> resultRows = null;

    if ( inputs.isEmpty() ) {
      resultRows = new ArrayList<>( 1 );
      resultRows.add( new Object[]{} );
      return resultRows;
    }

    int rowCount = inputs.stream().max( Comparator.comparingInt( List::size ) ).get().size();

    resultRows = new ArrayList<>( rowCount );

    Object[] resultRow = null;
    for ( int rownum = 0; rownum < rowCount; rownum++ ) {
      resultRow = new Object[ inputs.size() ];
      for ( int col = 0; col < inputs.size(); col++ ) {
        if ( inputs.get( col ).size() == 0 ) {
          resultRow[ col ] = null;
          continue;
        }
        resultRow[ col ] = inputs.get( col ).get( rownum );
      }
      resultRows.add( resultRow );
    }

    return resultRows;
  }

  private void filterOutExcessRows( List<Object[]> resultRows ) {
    boolean atLeastOneNonNull = resultRows.stream().anyMatch( el -> Arrays.stream( el ).anyMatch( in -> in != null ) );

    if ( !atLeastOneNonNull ) {
      resultRows.clear();
      resultRows.add( new Object[]{} );
      return;
    }

    resultRows.removeIf( el -> Arrays.stream( el ).allMatch( in -> in == null ) );

  }

  private void raiseExceptionIfAnyMissingPath( List<Object[]> resultRows ) throws JsonInputException {
    for ( Object[] resultRow : resultRows ) {
      for ( int i = 0; i < resultRow.length; i++ ) {
        if ( resultRow[i] == null ) {
          throw new JsonInputException( BaseMessages.getString( PKG, "JsonReader.Error.CanNotFindPath", fields[i].getPath() ) );
        }
      }
    }
  }

  public static boolean isAllNull( Iterable<?> list ) {
    for ( Object obj : list ) {
      if ( obj != null ) {
        return false;
      }
    }
    return true;
  }

}