/*******************************************************************************
* Copyright 2017 Capital One Services, LLC and Bitwise, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*******************************************************************************/
package hydrograph.engine.cascading.assembly;
import cascading.flow.FlowDef;
import cascading.pipe.Pipe;
import cascading.scheme.Scheme;
import cascading.tap.Tap;
import cascading.tap.hadoop.Hfs;
import cascading.tuple.Fields;
import hydrograph.engine.cascading.assembly.base.BaseComponent;
import hydrograph.engine.cascading.assembly.infra.ComponentParameters;
import hydrograph.engine.cascading.assembly.utils.InputOutputFieldsAndTypesCreator;
import hydrograph.engine.cascading.scheme.TextDelimitedAndFixedWidth;
import hydrograph.engine.core.component.entity.InputFileMixedSchemeEntity;
import hydrograph.engine.core.component.entity.elements.OutSocket;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class InputFileMixedSchemeAssembly extends BaseComponent<InputFileMixedSchemeEntity> {
private static final long serialVersionUID = 7857965970250755857L;
private static Logger LOG = LoggerFactory.getLogger(InputFileMixedSchemeAssembly.class);
Pipe pipe;
@SuppressWarnings("rawtypes")
Tap tap;
@SuppressWarnings("rawtypes")
Scheme scheme;
FlowDef flowDef;
private InputFileMixedSchemeEntity inputFileMixedSchemeEntity;
private InputOutputFieldsAndTypesCreator<InputFileMixedSchemeEntity> fieldsCreator;
public InputFileMixedSchemeAssembly(InputFileMixedSchemeEntity baseComponentEntity,
ComponentParameters componentParameters) {
super(baseComponentEntity, componentParameters);
}
@Override
protected void createAssembly() {
fieldsCreator = new InputOutputFieldsAndTypesCreator<InputFileMixedSchemeEntity>(inputFileMixedSchemeEntity);
try {
generateTapsAndPipes();
flowDef = flowDef.addSource(pipe, tap);
if (LOG.isTraceEnabled()) {
LOG.trace(inputFileMixedSchemeEntity.toString());
}
for (OutSocket outSocket : inputFileMixedSchemeEntity.getOutSocketList()) {
LOG.trace("Creating input file mixed scheme assembly for '"
+ inputFileMixedSchemeEntity.getComponentId() + "' for socket: '" + outSocket.getSocketId()
+ "' of type: '" + outSocket.getSocketType() + "'");
setOutLink(outSocket.getSocketType(), outSocket.getSocketId(),
inputFileMixedSchemeEntity.getComponentId(), pipe, scheme.getSourceFields());
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("unchecked")
public void generateTapsAndPipes() throws IOException {
try {
prepareScheme();
} catch (Exception e) {
LOG.error("Error in preparing scheme for component '" + inputFileMixedSchemeEntity.getComponentId() + "': "
+ e.getMessage());
throw new RuntimeException(e);
}
flowDef = componentParameters.getFlowDef();
// initializing each pipe and tap
tap = new Hfs(scheme, inputFileMixedSchemeEntity.getPath());
pipe = new Pipe(inputFileMixedSchemeEntity.getComponentId()+inputFileMixedSchemeEntity.getOutSocketList().get(0).getSocketId());
setHadoopProperties(tap.getStepConfigDef());
setHadoopProperties(pipe.getStepConfigDef());
}
public void prepareScheme() {
Fields inputFields = fieldsCreator.makeFieldsWithTypes();
scheme = new TextDelimitedAndFixedWidth(inputFields, fieldsCreator.getFieldLengthOrDelimiter(),
fieldsCreator.getTypeFieldLengthDelimiter(), inputFields.getTypes(),
inputFileMixedSchemeEntity.getStrict(), inputFileMixedSchemeEntity.getSafe(),
inputFileMixedSchemeEntity.getCharset(),inputFileMixedSchemeEntity.getQuote());
}
@Override
public void initializeEntity(InputFileMixedSchemeEntity assemblyEntityBase) {
this.inputFileMixedSchemeEntity=assemblyEntityBase;
}
}