/*******************************************************************************
* Copyright 2017 Capital One Services, LLC and Bitwise, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*******************************************************************************/
package hydrograph.engine.cascading.assembly;
import cascading.pipe.GroupBy;
import cascading.pipe.Pipe;
import cascading.tuple.Fields;
import hydrograph.engine.cascading.assembly.base.BaseComponent;
import hydrograph.engine.cascading.assembly.infra.ComponentParameters;
import hydrograph.engine.core.component.entity.SortEntity;
import hydrograph.engine.core.component.entity.elements.KeyField;
import hydrograph.engine.core.component.entity.elements.OutSocket;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
/**
* @author Prabodh
*/
public class SortAssembly extends BaseComponent<SortEntity> {
private static final long serialVersionUID = 3468787324609517976L;
private SortEntity sortEntity;
private static Logger LOG = LoggerFactory.getLogger(SortAssembly.class);
public SortAssembly(SortEntity baseComponentEntity, ComponentParameters componentParameters) {
super(baseComponentEntity, componentParameters);
}
@Override
protected void createAssembly() {
try {
if (LOG.isTraceEnabled()) {
LOG.trace(sortEntity.toString());
}
for (OutSocket outSocket : sortEntity.getOutSocketList()) {
LOG.trace("Creating sort assembly for '" + sortEntity.getComponentId() + "' for socket: '"
+ outSocket.getSocketId() + "' of type: '" + outSocket.getSocketType() + "'");
Fields keyFields = getFieldsFromKeyFields(sortEntity.getKeyFields());
Fields secondaryKeyFields = getFieldsFromKeyFields(sortEntity.getSecondaryKeyFields());
if (keyFields != null && LOG.isDebugEnabled()) {
LOG.debug("Key fields for sort component: '" + sortEntity.getComponentId() + "': "
+ keyFields.toString());
}
if (secondaryKeyFields != null && LOG.isDebugEnabled()) {
LOG.debug("Secondary key fields for sort component: '" + sortEntity.getComponentId() + "': "
+ secondaryKeyFields.toString());
}
Pipe sortOutput = new GroupBy(sortEntity.getComponentId()+outSocket.getSocketId(), componentParameters.getInputPipe(),
keyFields, secondaryKeyFields);
setHadoopProperties(sortOutput.getStepConfigDef());
setOutLink(outSocket.getSocketType(), outSocket.getSocketId(), sortEntity.getComponentId(), sortOutput,
componentParameters.getInputFields());
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage());
}
}
/**
* Creates an object of type {@link Fields} from array of {@link KeyField}
*
* @param keyFields
* an array of {@link KeyField} containing the field name and
* sort order
* @return an object of type {@link Fields}
*/
private Fields getFieldsFromKeyFields(KeyField[] keyFields) {
if (keyFields == null) {
return null;
}
String[] fieldNames = new String[keyFields.length];
int i = 0;
for (KeyField eachField : keyFields) {
fieldNames[i] = eachField.getName();
i++;
}
Fields fields = new Fields(fieldNames);
i = 0;
for (KeyField eachField : keyFields) {
if (eachField.getSortOrder().equalsIgnoreCase("desc")) {
fields.setComparator(eachField.getName(), Collections.reverseOrder());
}
i++;
}
return fields;
}
@Override
public void initializeEntity(SortEntity assemblyEntityBase) {
this.sortEntity=assemblyEntityBase;
}
}