/**
 * This software is licensed to you under the Apache License, Version 2.0 (the
 * "Apache License").
 *
 * LinkedIn's contributions are made under the Apache License. If you contribute
 * to the Software, the contributions will be deemed to have been made under the
 * Apache License, unless you expressly indicate otherwise. Please do not make any
 * contributions that would be inconsistent with the Apache License.
 *
 * You may obtain a copy of the Apache License at http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, this software
 * distributed under the Apache License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the Apache
 * License for the specific language governing permissions and limitations for the
 * software governed under the Apache License.
 *
 * © 2012 LinkedIn Corp. All Rights Reserved.  
 */
package com.senseidb.indexing.hadoop.reduce;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Closeable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.log4j.Logger;

import com.senseidb.indexing.hadoop.keyvalueformat.IntermediateForm;
import com.senseidb.indexing.hadoop.keyvalueformat.Shard;
import com.senseidb.indexing.hadoop.util.MRConfig;

public class SenseiReducer extends MapReduceBase implements
		Reducer<Shard, IntermediateForm, Shard, Text> {
	
	private static final Logger logger = Logger.getLogger(SenseiReducer.class);
	static final Text DONE = new Text("done");
	

	private Configuration iconf;
	private String mapredTempDir;
	  
    public void reduce(Shard key, Iterator<IntermediateForm> values,
                       OutputCollector<Shard, Text> output, 
                       Reporter reporter) throws IOException {

    	logger.info("Construct a shard writer for " + key);
        FileSystem fs = FileSystem.get(iconf);
        //debug:
        logger.info("filesystem is: "+ fs.getName());
        String temp = mapredTempDir + Path.SEPARATOR + "shard_" + key.toFlatString()+ "_" + System.currentTimeMillis();
        logger.info("mapredTempDir is: "+ mapredTempDir);
        final ShardWriter writer = new ShardWriter(fs, key, temp, iconf);

        // update the shard
        while (values.hasNext()) {
          IntermediateForm form = values.next();
          writer.process(form);
          reporter.progress();
        }

        // close the shard
        final Reporter fReporter = reporter;
        new Closeable() {
          volatile boolean closed = false;

          public void close() throws IOException {
            // spawn a thread to give progress heartbeats
            Thread prog = new Thread() {
              public void run() {
                while (!closed) {
                  try {
                    fReporter.setStatus("closing");
                    Thread.sleep(1000);
                  } catch (InterruptedException e) {
                    continue;
                  } catch (Throwable e) {
                    return;
                  }
                }
              }
            };

            try {
              prog.start();

              if (writer != null) {
            	writer.optimize();  //added this option to optimize after all the docs have been added;
                writer.close();
              }
            } finally {
              closed = true;
            }
          }
        }.close();
        logger.info("Closed the shard writer for " + key + ", writer = " + writer);
    	
    	
      output.collect(key, DONE);
    }
    
    @Override
    public void close() throws IOException {
    	if(mapredTempDir != null){
    		File file = new File(mapredTempDir);
    		if(file.exists())
    			deleteDir(file);
    	}
    }
    
    static void deleteDir(File file)
    {
      if (file == null || !file.exists())
        return;
      for (File f : file.listFiles())
      {
        if (f.isDirectory())
          deleteDir(f);
        else
          f.delete();
      }
      file.delete();
    }
    
    public void configure(JobConf job) {
        iconf = job;
        mapredTempDir = iconf.get(MRConfig.TEMP_DIR);
        mapredTempDir = Shard.normalizePath(mapredTempDir);
      }
  }