/******************************************************************************* * Copyright 2017 Capital One Services, LLC and Bitwise, Inc. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * http://www.apache.org/licenses/LICENSE-2.0 * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License *******************************************************************************/ package hydrograph.engine.cascading.assembly.base; import hydrograph.engine.cascading.assembly.infra.ComponentParameters; import hydrograph.engine.core.component.entity.base.AssemblyEntityBase; import hydrograph.engine.core.constants.ComponentLinkType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import cascading.pipe.Pipe; import cascading.pipe.SubAssembly; import cascading.property.ConfigDef; import cascading.tuple.Fields; /** * @author gurdits * */ public abstract class BaseComponent<T extends AssemblyEntityBase> extends SubAssembly { private static final long serialVersionUID = -804998221607089402L; private HashMap<String, HashMap<String, Pipe>> outLinksMap; private HashMap<String, HashMap<String, Fields>> outLinksFields; private AssemblyEntityBase baseComponentEntity; protected ComponentParameters componentParameters; private Map<String, Fields> outFieldsMap; private static Logger LOG = LoggerFactory.getLogger(BaseComponent.class); public BaseComponent(T baseComponentEntity, ComponentParameters componentParameters) { this.outLinksMap = new HashMap<String, HashMap<String, Pipe>>(); this.outLinksFields = new HashMap<String, HashMap<String, Fields>>(); this.outFieldsMap = new HashMap<String, Fields>(); this.baseComponentEntity = baseComponentEntity; this.componentParameters = componentParameters; if (LOG.isTraceEnabled()) { logComponentParameterInfo(); } initializeEntity(baseComponentEntity); createAssembly(); setOutLinksToTails(); } private void logComponentParameterInfo() { StringBuilder str = new StringBuilder( "Component parameter information for component: " + baseComponentEntity.getComponentId()); str.append("\nIn socket: "); if (componentParameters.getinSocketId() != null) { str.append(Arrays.toString(componentParameters.getinSocketId() .toArray())); } str.append("\nInput fields: "); if (componentParameters.getInputFields() != null) { str.append(Arrays.toString(componentParameters.getInputFieldsList() .toArray())); } str.append("\nOutput fields: "); if (componentParameters.getOutputFieldsList() != null) { str.append(Arrays.toString(componentParameters .getOutputFieldsList().toArray())); } LOG.trace(str.toString()); } /** * Casts the {@link AssemblyEntityBase} object to the object of entity class * specific to the assembly * * @param assemblyEntityBase * the {@link AssemblyEntityBase} object to cast to specific * entity object */ public abstract void initializeEntity(T assemblyEntityBase); protected abstract void createAssembly(); protected void setOutLink(String socketType, String socketId, String componentId, Pipe pipeLink, Fields outputFields) { // add pipe link HashMap<String, Pipe> linkMap = outLinksMap.get(socketType); // If we have not dealt with particular type of link so far then add it if (linkMap == null) { linkMap = new HashMap<String, Pipe>(); outLinksMap.put(socketId, linkMap); } linkMap.put(componentId + "_" + socketId, pipeLink); // add link output schema HashMap<String, Fields> fieldsMap = outLinksFields.get(socketType); // If we have not dealt with particular type of link so far then add it if (fieldsMap == null) { fieldsMap = new HashMap<String, Fields>(); outLinksFields.put(socketId, fieldsMap); } fieldsMap.put(componentId + "_" + socketId, outputFields); outFieldsMap.put(componentId + "_" + socketId, outputFields); } public Pipe getOutLink(String socketType, String socketId, String componentId) { HashMap<String, Pipe> linkMap = outLinksMap.get(socketId); // If we have not dealt with particular type of link so far then add it if (linkMap == null) { return null; } return linkMap.get(componentId + "_" + socketId); } public Fields getOutFields(String socketType, String socketId, String componentId) { HashMap<String, Fields> fieldsMap = outLinksFields.get(socketId); if (fieldsMap == null) { return null; } return fieldsMap.get(componentId + "_" + socketId); } protected HashMap<String, Pipe> getAllOutLink(ComponentLinkType linkType) { return outLinksMap.get(linkType.toString()); } public Map<String, Fields> getAllOutFields() { return outFieldsMap; } public Collection<HashMap<String, Pipe>> getAllOutLinkForAssembly() { return outLinksMap.values(); } private void setOutLinksToTails() { ArrayList<Pipe> allOutPipes = new ArrayList<Pipe>(); // traverse entire data structure to extract all damn pipes for (HashMap<String, Pipe> linkMap : outLinksMap.values()) { for (Pipe outPipes : linkMap.values()) { allOutPipes.add(outPipes); } } // set tails only if there is something to set if (allOutPipes.size() > 0) { setTails(allOutPipes.toArray(new Pipe[allOutPipes.size()])); } } protected void setHadoopProperties(ConfigDef conf) { Properties hadoopProps = baseComponentEntity.getRuntimeProperties(); if (hadoopProps != null) { // Iterate over all hadoop props and set it for current component for (Map.Entry<Object, Object> entry : hadoopProps.entrySet()) { conf.setProperty(ConfigDef.Mode.REPLACE, (String) entry.getKey(), (String) entry.getValue()); } } } }