HDFSmaterializer.java

/*
 * Copyright (c) 2011-2015 EPFL DATA Laboratory
 * Copyright (c) 2014-2015 The Squall Collaboration (see NOTICE)
 *
 * All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package ch.epfl.data.squall.connectors.hdfs;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.apache.commons.lang.ArrayUtils;
import org.apache.storm.hdfs.bolt.HdfsBolt;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
import org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.format.RecordFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;

import backtype.storm.Config;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.topology.base.BaseRichBolt;
import ch.epfl.data.squall.components.Component;
import ch.epfl.data.squall.components.DataSourceComponent;
import ch.epfl.data.squall.expressions.ValueExpression;
import ch.epfl.data.squall.operators.ChainOperator;
import ch.epfl.data.squall.operators.Operator;
import ch.epfl.data.squall.predicates.Predicate;
import ch.epfl.data.squall.storm_components.StormComponent;
import ch.epfl.data.squall.storm_components.StormEmitter;
import ch.epfl.data.squall.storm_components.synchronization.TopologyKiller;
import ch.epfl.data.squall.types.Type;
import ch.epfl.data.squall.utilities.MyUtilities;

public class HDFSmaterializer implements Component {

    private final String _componentName;

    private long _batchOutputMillis;

    private List<Integer> _hashIndexes;
    private List<ValueExpression> _hashExpressions;

    private final ChainOperator _chain = new ChainOperator();

    private boolean _printOut;
    private boolean _printOutSet;

    private final Component _parent;
    private Component _child;
    // private StormOperator _stormOperator;

    private List<String> _fullHashList;

    private String _hdfsPath;
    private String _folderName;
    private int _parallelism;

    public HDFSmaterializer(Component parent, String componentName,
	    String hdfsPath, String folderName, int parallelism) {

	_parent = parent;
	_parent.setChild(this);
	_componentName = componentName;
	_hdfsPath = hdfsPath;
	_folderName = folderName;
	_parallelism = parallelism;
    }

    public BaseRichBolt createHDFSmaterializer(String hdfsPath) {
	RecordFormat format = new DelimitedRecordFormat()
		.withFieldDelimiter("|");

	// sync the filesystem after every 1k tuples
	SyncPolicy syncPolicy = new CountSyncPolicy(1000);

	// rotate files when they reach 5MB
	FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f,
		Units.MB);

	FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(
		"/" + _folderName + "/").withExtension(".txt");

	HdfsBolt bolt = new HdfsBolt().withFsUrl(hdfsPath)
		.withFileNameFormat(fileNameFormat).withRecordFormat(format)
		.withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
	return bolt;
    }

    @Override
    public HDFSmaterializer add(Operator operator) {
	_chain.addOperator(operator);
	return this;
    }

    @Override
    public boolean equals(Object obj) {
	if (obj instanceof Component)
	    return _componentName.equals(((Component) obj).getName());
	else
	    return false;
    }

    @Override
    public List<DataSourceComponent> getAncestorDataSources() {
	final List<DataSourceComponent> list = new ArrayList<DataSourceComponent>();
	list.addAll(_parent.getAncestorDataSources());
	return list;
    }

    @Override
    public long getBatchOutputMillis() {
	return _batchOutputMillis;
    }

    @Override
    public ChainOperator getChainOperator() {
	return _chain;
    }

    @Override
    public Component getChild() {
	return _child;
    }

    // from StormComponent
    @Override
    public String[] getEmitterIDs() {
	throw new RuntimeException("Not implemented yet");
    }

    @Override
    public List<String> getFullHashList() {
	return _fullHashList;
    }

    @Override
    public List<ValueExpression> getHashExpressions() {
	return _hashExpressions;
    }

    @Override
    public List<Integer> getHashIndexes() {
	return _hashIndexes;
    }

    @Override
    public String getInfoID() {
	throw new RuntimeException("Not implemented yet");
    }

    @Override
    public String getName() {
	return _componentName;
    }

    @Override
    public Component[] getParents() {
	return new Component[] { _parent };
    }

    @Override
    public boolean getPrintOut() {
	return _printOut;
    }

    @Override
    public int hashCode() {
	int hash = 5;
	hash = 47 * hash
		+ (_componentName != null ? _componentName.hashCode() : 0);
	return hash;
    }

    @Override
    public void makeBolts(TopologyBuilder builder, TopologyKiller killer,
	    List<String> allCompNames, Config conf, int hierarchyPosition) {

	// by default print out for the last component
	// for other conditions, can be set via setPrintOut
	if (hierarchyPosition == StormComponent.FINAL_COMPONENT
		&& !_printOutSet)
	    setPrintOut(true);

	MyUtilities.checkBatchOutput(_batchOutputMillis,
		_chain.getAggregation(), conf);

	BaseRichBolt hdfsBolt = createHDFSmaterializer(_hdfsPath);

	builder.setBolt("hdfs", hdfsBolt, _parallelism).shuffleGrouping(
		((StormEmitter) _parent).getEmitterIDs()[0]);

    }

    @Override
    public HDFSmaterializer setBatchOutputMillis(long millis) {
	_batchOutputMillis = millis;
	return this;
    }

    @Override
    public void setChild(Component child) {
	_child = child;
    }

    @Override
    public Component setContentSensitiveThetaJoinWrapper(Type wrapper) {
	return this;
    }

    @Override
    public HDFSmaterializer setFullHashList(List<String> fullHashList) {
	_fullHashList = fullHashList;
	return this;
    }

    @Override
    public HDFSmaterializer setHashExpressions(
	    List<ValueExpression> hashExpressions) {
	_hashExpressions = hashExpressions;
	return this;
    }

    @Override
    public HDFSmaterializer setOutputPartKey(int... hashIndexes) {
	return setOutputPartKey(Arrays.asList(ArrayUtils.toObject(hashIndexes)));
    }

    @Override
    public HDFSmaterializer setOutputPartKey(List<Integer> hashIndexes) {
	_hashIndexes = hashIndexes;
	return this;
    }

    @Override
    public HDFSmaterializer setPrintOut(boolean printOut) {
	_printOutSet = true;
	_printOut = printOut;
	return this;
    }

}