DownloaderReducer.java

package org.hipi.tools.downloader;

import org.hipi.imagebundle.HipiImageBundle;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class DownloaderReducer extends Reducer<BooleanWritable, Text, BooleanWritable, Text> {
  
  private static Configuration conf;
  
  @Override
  public void setup(Context context) throws IOException, InterruptedException {
    this.conf = context.getConfiguration();
  }
  
  // Combine HIBs produced by the map tasks into a single HIB
  @Override
  public void reduce(BooleanWritable key, Iterable<Text> values, Context context)
    throws IOException, InterruptedException {
    
    if (key.get()) {
      
      // Get path to output HIB
      FileSystem fileSystem = FileSystem.get(conf);
      Path outputHibPath = new Path(conf.get("downloader.outfile"));
      
      // Create HIB for writing
      HipiImageBundle hib = new HipiImageBundle(outputHibPath, conf);
      hib.openForWrite(true);
      
      // Iterate over the temporary HIB files created by map tasks
      for (Text tempString : values) {
	
	// Open the temporary HIB file
	Path tempPath = new Path(tempString.toString());
	HipiImageBundle inputBundle = new HipiImageBundle(tempPath, conf);
	
	// Append temporary HIB file to output HIB (this is fast)
	hib.append(inputBundle);
	
	// Remove temporary HIB (both .hib and .hib.dat files)
	Path indexPath = inputBundle.getPath();
	Path dataPath = new Path(indexPath.toString() + ".dat");
	fileSystem.delete(indexPath, false);
	fileSystem.delete(dataPath, false);
	
	// Emit output key/value pair indicating temporary HIB has been processed
	Text outputPath = new Text(inputBundle.getPath().toString());
	context.write(new BooleanWritable(true), outputPath);
	context.progress();
      }
      
      // Finalize output HIB
      hib.close();
    }
  }

}