/*******************************************************************************
* Copyright 2017 Capital One Services, LLC and Bitwise, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*******************************************************************************/
package hydrograph.engine.cascading.assembly;
import cascading.flow.FlowDef;
import cascading.pipe.Pipe;
import cascading.scheme.Scheme;
import cascading.tap.Tap;
import cascading.tap.hadoop.Hfs;
import cascading.tuple.Fields;
import hydrograph.engine.cascading.assembly.base.BaseComponent;
import hydrograph.engine.cascading.assembly.infra.ComponentParameters;
import hydrograph.engine.cascading.assembly.utils.InputOutputFieldsAndTypesCreator;
import hydrograph.engine.cascading.scheme.TextFixedWidth;
import hydrograph.engine.core.component.entity.InputFileFixedWidthEntity;
import hydrograph.engine.core.component.entity.elements.OutSocket;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
public class InputFileFixedWidthAssembly extends BaseComponent<InputFileFixedWidthEntity> {
/**
*
*/
private static final long serialVersionUID = 8145806669418685707L;
InputFileFixedWidthEntity inputFileFixedWidthEntity;
Pipe pipe;
@SuppressWarnings("rawtypes")
Tap tap;
@SuppressWarnings("rawtypes")
Scheme scheme;
FlowDef flowDef;
private static Logger LOG = LoggerFactory
.getLogger(InputFileFixedWidthAssembly.class);
private InputOutputFieldsAndTypesCreator<InputFileFixedWidthEntity> fieldsCreator;
public InputFileFixedWidthAssembly(InputFileFixedWidthEntity baseComponentEntity,
ComponentParameters componentParameters) {
super(baseComponentEntity, componentParameters);
}
@Override
protected void createAssembly() {
try {
fieldsCreator = new InputOutputFieldsAndTypesCreator<InputFileFixedWidthEntity>(
inputFileFixedWidthEntity);
LOG.debug("InputFile Fixed Width Component: [ Fields List : " + Arrays.toString(fieldsCreator.getFieldNames()) + ", Field Types : "
+ Arrays.toString(fieldsCreator.getFieldDataTypes()) + ", Field Length : " + Arrays.toString(fieldsCreator.getFieldLength())
+ " , Path : " + inputFileFixedWidthEntity.getPath() + ", Batch : "
+ inputFileFixedWidthEntity.getBatch() + "]");
generateTapsAndPipes();
flowDef = flowDef.addSource(pipe, tap);
if (LOG.isTraceEnabled()) {
LOG.trace(inputFileFixedWidthEntity.toString());
}
for (OutSocket outSocket : inputFileFixedWidthEntity
.getOutSocketList()) {
LOG.trace("Creating input file fixed width assembly for '"
+ inputFileFixedWidthEntity.getComponentId()
+ "' for socket: '" + outSocket.getSocketId()
+ "' of type: '" + outSocket.getSocketType() + "'");
setOutLink(outSocket.getSocketType(), outSocket.getSocketId(),
inputFileFixedWidthEntity.getComponentId(), pipe,
scheme.getSourceFields());
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage());
}
}
@SuppressWarnings("unchecked")
public void generateTapsAndPipes() throws IOException {
try{
prepareScheme();
}
catch(Exception e) {
LOG.error("Error in preparing scheme for component '"
+ inputFileFixedWidthEntity.getComponentId() + "': " + e.getMessage());
throw new RuntimeException(e);
}
flowDef = componentParameters.getFlowDef();
// initializing each pipe and tap
tap = new Hfs(scheme, inputFileFixedWidthEntity.getPath());
pipe = new Pipe(inputFileFixedWidthEntity.getComponentId()+inputFileFixedWidthEntity.getOutSocketList().get(0).getSocketId());
setHadoopProperties(tap.getStepConfigDef());
setHadoopProperties(pipe.getStepConfigDef());
}
protected void prepareScheme() {
int[] fieldLength = new int[inputFileFixedWidthEntity.getFieldsList()
.size()];
for (int i = 0; i < inputFileFixedWidthEntity.getFieldsList().size(); i++) {
fieldLength[i] = inputFileFixedWidthEntity.getFieldsList().get(i)
.getFieldLength();
}
Fields inputFields = fieldsCreator.makeFieldsWithTypes();
scheme = new TextFixedWidth(inputFields, fieldLength, null,
inputFileFixedWidthEntity.isStrict(),
inputFileFixedWidthEntity.isSafe(),
inputFileFixedWidthEntity.getCharset());
}
@Override
public void initializeEntity(InputFileFixedWidthEntity assemblyEntityBase) {
this.inputFileFixedWidthEntity=assemblyEntityBase;
}
}