SolbaseIndexReducer.java

package org.solbase.indexer.mapreduce;

import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Iterator;
import java.util.List;

import javax.management.RuntimeErrorException;

import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Writable;
import org.apache.lucene.index.Term;
import org.solbase.SolbaseUtil;
import org.solbase.indexer.writable.DocumentPutWritable;
import org.solbase.indexer.writable.TermDocMetadataWritable;
import org.solbase.lucenehbase.TermDocMetadataLoader;

public class SolbaseIndexReducer extends TableReducer<BytesWritable, MapWritable, Writable> {
	
	public static enum Counters {
		TOTAL_DOCS,TOTAL_TERM_VECTORS,TOTAL_TERM_VECTOR_VERSION,TOTAL_DOC_KEY_ID_MAP,TOTAL_INVALID,DUPLICATE_ROWS
	}
	
	public void reduce(BytesWritable key, Iterable<MapWritable> values, Context context) throws IOException, InterruptedException {
		
		byte[] _key = null;
		int counter = 0;
		int dupCount = 0;
		
		// since key is checksum, we should do dedupping here
		// TODO: for now, i'm only retrieving one and ignoring rest
		boolean first = true;
		for (MapWritable writable: values){
			
			if(first){
				first = false;
				Iterator<Writable> itr = writable.keySet().iterator();

				while (itr.hasNext()) {
					BytesWritable wrtKey = (BytesWritable) itr.next();
					Writable wrt = writable.get(wrtKey);
					if (wrt instanceof DocumentPutWritable) {
						DocumentPutWritable docBytes = (DocumentPutWritable) wrt;

						String globalId = docBytes.getGlobalId();
						int docId = docBytes.getDocId();

						Put mapping = new Put(Bytes.toBytes(globalId));
						mapping.add(Bytes.toBytes("docId"), Bytes.toBytes(""), Bytes.toBytes(docId));
						context.write(new ImmutableBytesWritable(SolbaseUtil.docKeyIdMapTable), mapping);
						context.getCounter(Counters.TOTAL_DOC_KEY_ID_MAP).increment(1);

						List<String> fieldKeys = docBytes.getFieldKeys();
						List<byte[]> fieldValues = docBytes.getFieldValues();
						List<Term> allTerms = docBytes.getAllTerms();

						byte[] md5DocId = SolbaseUtil.randomize(docId);

						Put documentPut = new Put(md5DocId);

						// Store each field as a column under this docId
						for (int i = 0; i < fieldKeys.size(); i++) {
							String fieldKey = fieldKeys.get(i);
							byte[] fieldValue = fieldValues.get(i);

							documentPut.add(Bytes.toBytes("field"), Bytes.toBytes(fieldKey), fieldValue);
						}

						// Finally, Store meta-data so we can delete this
						// document
						documentPut.add(Bytes.toBytes("allTerms"), Bytes.toBytes("allTerms"), SolbaseUtil.toBytes(allTerms).array());

						context.write(new ImmutableBytesWritable(SolbaseUtil.docTable), documentPut);
						context.getCounter(Counters.TOTAL_DOCS).increment(1);

						counter++;

					} else if (wrt instanceof TermDocMetadataWritable) {
						// gather all of docs given field key (field/value)
						TermDocMetadataWritable metadata = (TermDocMetadataWritable) wrt;

						// convert key to byte array
						// byte[] fieldTermKey = key.getBytes();
						byte[] termValue = metadata.getTermDocMetadata();
						_key = metadata.getFieldTermKey();

						int docId = metadata.getDocId();

						Put put = null;

						switch (TermDocMetadataLoader.storageType) {
						case KEY_ONLY: {
							put = new Put(Bytes.add(Bytes.add(_key, SolbaseUtil.delimiter, Bytes.toBytes(docId)), termValue));
							put.add(SolbaseUtil.termVectorDocColumnFamilyName, Bytes.toBytes(""), Bytes.toBytes(""));
						}
							break;
						case WIDE_ROW:
							int chunkId = TermDocMetadataLoader.getChunkId(docId);
							put = new Put(Bytes.add(_key, SolbaseUtil.delimiter, Bytes.toBytes(chunkId)));
							put.add(SolbaseUtil.termVectorDocColumnFamilyName, Bytes.toBytes(docId), termValue);
							break;
						case NARROW_ROW:
						default: {
							put = new Put(Bytes.add(_key, SolbaseUtil.delimiter, Bytes.toBytes(docId)));
							put.add(SolbaseUtil.termVectorDocColumnFamilyName, Bytes.toBytes(""), termValue);
						}
						}

						context.write(new ImmutableBytesWritable(SolbaseUtil.termVectorTable), put);
						context.getCounter(Counters.TOTAL_TERM_VECTORS).increment(1);

						counter++;
					} else {
						System.out.println("else: " + writable.getClass());
						context.getCounter(Counters.TOTAL_INVALID).increment(1);
					}
				}
			} else {
				dupCount++;
			}
		}
		
		context.getCounter(Counters.DUPLICATE_ROWS).increment(dupCount);
	}
}