/*******************************************************************************
* Copyright 2017 Capital One Services, LLC and Bitwise, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*******************************************************************************/
package hydrograph.engine.cascading.assembly;
import cascading.pipe.Each;
import cascading.pipe.Pipe;
import cascading.tuple.Fields;
import hydrograph.engine.cascading.assembly.base.BaseComponent;
import hydrograph.engine.cascading.assembly.infra.ComponentParameters;
import hydrograph.engine.cascading.functions.CustomTuplesPartitioner;
import hydrograph.engine.core.component.entity.PartitionByExpressionEntity;
import hydrograph.engine.core.component.entity.elements.OutSocket;
import hydrograph.engine.transformation.userfunctions.base.CustomPartitionExpression;
import hydrograph.engine.utilities.UserClassLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.LinkedList;
public class PartitionByExpressionAssembly extends BaseComponent<PartitionByExpressionEntity> {
private PartitionByExpressionEntity partitionByExpressionEntity;
private static final long serialVersionUID = 8282067513240963488L;
private Pipe[] partitionedPipes;
private Pipe partitionInputPipe;
private CustomPartitionExpression partition;
private static Logger LOG = LoggerFactory.getLogger(PartitionByExpressionAssembly.class);
public PartitionByExpressionAssembly(PartitionByExpressionEntity baseComponentEntity,
ComponentParameters componentParameters) {
super(baseComponentEntity, componentParameters);
}
private void initAssembly() {
String className = partitionByExpressionEntity.getOperation().getOperationClass();
this.setPartition(UserClassLoader.loadAndInitClass(className, CustomPartitionExpression.class));
partitionedPipes = new Pipe[(int) partitionByExpressionEntity.getNumPartitions()];
partitionInputPipe = componentParameters.getInputPipe();
}
@Override
protected void createAssembly() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace(partitionByExpressionEntity.toString());
}
initAssembly();
createPartitionedPipes();
for (int i = 0; i < partitionedPipes.length; i++) {
setOutLink(partitionByExpressionEntity.getOutSocketList().get(i).getSocketType(),
partitionByExpressionEntity.getOutSocketList().get(i).getSocketId(),
partitionByExpressionEntity.getComponentId(), partitionedPipes[i],
componentParameters.getInputFields());
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage());
}
}
private LinkedList<String> getOutSocketIdList() {
LinkedList<String> outSocketIdList = new LinkedList<String>();
for (OutSocket outSocket : partitionByExpressionEntity.getOutSocketList()) {
outSocketIdList.add(outSocket.getSocketId());
}
return outSocketIdList;
}
private void createPartitionedPipes() {
int partitionIndex = 0;
for (String outSocketId : getOutSocketIdList()) {
partitionedPipes[partitionIndex] = getNewPipe(outSocketId, partitionInputPipe);
setHadoopProperties(partitionedPipes[partitionIndex].getStepConfigDef());
partitionIndex++;
}
}
private Pipe getNewPipe(String outSocketId, Pipe pipe) {
Pipe tempPipe = new Pipe(partitionByExpressionEntity.getComponentId()+outSocketId, pipe);
tempPipe = new Each(tempPipe, getNewPartitioner(outSocketId));
return tempPipe;
}
private CustomTuplesPartitioner getNewPartitioner(String outSocketId) {
return new CustomTuplesPartitioner(componentParameters.getInputFields(),
new Fields(partitionByExpressionEntity.getOperation().getOperationInputFields()), this.getPartition(),
(int) partitionByExpressionEntity.getNumPartitions(), outSocketId,
partitionByExpressionEntity.getOperation().getOperationProperties());
}
public CustomPartitionExpression getPartition() {
return this.partition;
}
public void setPartition(CustomPartitionExpression partition) {
this.partition = partition;
}
@Override
public void initializeEntity(PartitionByExpressionEntity assemblyEntityBase) {
this.partitionByExpressionEntity=assemblyEntityBase;
}
}