/*******************************************************************************
* Copyright 2017 Capital One Services, LLC and Bitwise, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License
*******************************************************************************/
package hydrograph.engine.utilites;
import cascading.flow.FlowDef;
import cascading.pipe.Pipe;
import cascading.scheme.Scheme;
import cascading.scheme.hadoop.TextDelimited;
import cascading.tap.SinkMode;
import cascading.tap.Tap;
import cascading.tap.hadoop.Hfs;
import cascading.tuple.Fields;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class AssemblyBuildHelper {
static Tap taps;
static Pipe pipes;
static FlowDef flow;
static int dummy = 0;
public static Pipe generateInputPipes(Fields fields, FlowDef flowDef, String filePath) {
Scheme scheme = new TextDelimited(fields, true, ",");
flow = flowDef;
taps = new Hfs(scheme, filePath);
pipes = new Pipe(String.valueOf(Math.random()));
flow.addSource(pipes, taps);
return pipes;
}
public static Pipe generateInputPipes(Fields fields, FlowDef flowDef, String filePath, boolean hasHeader) {
Scheme scheme = new TextDelimited(fields, false, ",");
flow = flowDef;
taps = new Hfs(scheme, filePath);
pipes = new Pipe(String.valueOf(Math.random()));
flow.addSource(pipes, taps);
return pipes;
}
public static Pipe generateInputPipes(Fields fields, String Delimiter, Boolean Header, FlowDef flowDef,
String filePath) {
Scheme scheme = new TextDelimited(fields, Header, Delimiter);
flow = flowDef;
taps = new Hfs(scheme, filePath);
pipes = new Pipe("InputPipe");
flow.addSource(pipes, taps);
return pipes;
}
public static void generateOutputPipes(Fields fields, Pipe pipes, String filePath, FlowDef flowDef) {
Scheme scheme = new TextDelimited(fields, true, ",");
flow = flowDef;
taps = new Hfs(scheme, filePath + pipes.getName(), SinkMode.REPLACE);
flow.addTailSink(pipes, taps);
}
public static void generateOutputPipes(Pipe pipes, String filePath, FlowDef flowDef) {
Scheme scheme = new TextDelimited(true, ",");
flow = flowDef;
taps = new Hfs(scheme, filePath + "/" + pipes.getName(), SinkMode.REPLACE);
flow.addTailSink(pipes, taps);
}
public static FlowDef getFlow() {
return flow;
}
public static Tap generateOutTaps(String filePath) {
Scheme scheme = new TextDelimited(true, ",");
taps = new Hfs(scheme, filePath, SinkMode.REPLACE);
return taps;
}
/*
* public static Properties setProperty(ComponentParameters params) {
* Properties props = new Properties();
* props.put("cascading.compatibility.retain.collector", "true");
* params.addHadoopLocalProperties(props); return props;
*
* }
*/
}