TermReduce.java

package com.yahoo.glimmer.indexing.generator;

/*
 * Copyright (c) 2012 Yahoo! Inc. All rights reserved.
 * 
 *  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 *  Unless required by applicable law or agreed to in writing, software distributed under the License is 
 *  distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and limitations under the License.
 *  See accompanying LICENSE file.
 */

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Reducer;

import com.yahoo.glimmer.indexing.generator.TermValue.Type;

public class TermReduce extends Reducer<TermKey, TermValue, IntWritable, IndexRecordWriterValue> {
    private static final Log LOG = LogFactory.getLog(TermReduce.class);
    public static final String MAX_INVERTEDLIST_SIZE_PARAMETER = "maxInvertiedListSize";
    public static final String MAX_POSITIONLIST_SIZE_PARAMETER = "maxPositionListSize";

    private IntWritable writerKey;
    private IndexRecordWriterTermValue writerTermValue;
    private IndexRecordWriterDocValue writerDocValue;
    private IndexRecordWriterSizeValue writerSizeValue;
    private ArrayList<Long> predicatedIds;
    private long termKeysProcessed;

    @Override
    protected void setup(org.apache.hadoop.mapreduce.Reducer<TermKey, TermValue, IntWritable, IndexRecordWriterValue>.Context context) throws IOException,
	    InterruptedException {
	writerKey = new IntWritable();
	writerTermValue = new IndexRecordWriterTermValue();
	writerDocValue = new IndexRecordWriterDocValue();
	writerSizeValue = new IndexRecordWriterSizeValue();
	predicatedIds = new ArrayList<Long>();
    };

    @Override
    public void reduce(TermKey key, Iterable<TermValue> values, Context context) throws IOException, InterruptedException {
	if (key == null || key.equals("")) {
	    return;
	}

	if (termKeysProcessed % 10000 == 0) {
	    String statusString = "Reducing " + key.toString();
	    context.setStatus(statusString);
	    LOG.info(statusString);
	}

	writerKey.set(key.getIndex());

	if (key.getIndex() == DocumentMapper.ALIGNMENT_INDEX) {
	    long lastPredicateId = Long.MIN_VALUE;
	    for (TermValue value : values) {
		if (value.getType() != Type.INDEX_ID) {
		    throw new IllegalStateException("Got a " + value.getType() + " value when expecting only " + Type.INDEX_ID);
		}
		if (lastPredicateId != value.getV1()) {
		    lastPredicateId = value.getV1();
		    predicatedIds.add(lastPredicateId);
		}
	    }

	    writerTermValue.setTerm(key.getTerm());
	    writerTermValue.setOccurrenceCount(0);
	    writerTermValue.setTermFrequency(predicatedIds.size());
	    writerTermValue.setSumOfMaxTermPositions(0);

	    context.write(writerKey, writerTermValue);

	    for (Long predicateId : predicatedIds) {
		writerDocValue.setDocument(predicateId);
		context.write(writerKey, writerDocValue);
	    }
	    predicatedIds.clear();
	} else if (TermKey.DOC_SIZE_TERM.equals(key.getTerm())) {
	    // Write .sizes files
	    Iterator<TermValue> valuesIt = values.iterator();
	    while (valuesIt.hasNext()) {
		TermValue value = valuesIt.next();

		if (Type.DOC_SIZE != value.getType()) {
		    throw new IllegalStateException("Got a " + value.getType() + " value when expecting only " + Type.DOC_SIZE);
		}

		writerSizeValue.setDocument(value.getV1());
		writerSizeValue.setSize(value.getV2());
		context.write(writerKey, writerSizeValue);
	    }
	} else {
	    int termFrequency = 0;
	    int termCount = 0;
	    int sumOfMaxTermPositions = 0;
	    TermValue value = null;

	    Iterator<TermValue> valuesIt = values.iterator();
	    while (valuesIt.hasNext()) {
		value = valuesIt.next();

		if (Type.TERM_STATS != value.getType()) {
		    break;
		}
		termFrequency++;
		termCount += value.getV1();
		sumOfMaxTermPositions += value.getV2();
	    }

	    if (Type.OCCURRENCE != value.getType()) {
		throw new IllegalStateException("Got a " + value.getType() + " value when expecting only " + Type.OCCURRENCE);
	    }

	    writerTermValue.setTerm(key.getTerm());
	    writerTermValue.setOccurrenceCount(termCount);
	    writerTermValue.setTermFrequency(termFrequency);
	    writerTermValue.setSumOfMaxTermPositions(sumOfMaxTermPositions);

	    context.write(writerKey, writerTermValue);

	    TermValue prevValue = new TermValue();
	    prevValue.set(value);

	    while (value != null && value.getType() == Type.OCCURRENCE) {
		long docId = value.getV1();
		if (docId < 0) {
		    throw new IllegalStateException("Negative DocID. Key:" + key + "\nValue:" + value);
		}
		if (docId != prevValue.getV1()) {
		    // New document, write out previous postings
		    writerDocValue.setDocument(prevValue.getV1());

		    context.write(writerKey, writerDocValue);

		    // The first occerrence of this docId/
		    writerDocValue.clearOccerrences();
		    writerDocValue.addOccurrence(value.getV2());
		} else {
		    writerDocValue.addOccurrence(value.getV2());
		}

		prevValue.set(value);

		boolean last = false;
		if (valuesIt.hasNext()) {
		    value = valuesIt.next();
		    // LOG.warn("Value:" + value.toString());
		    // Skip equivalent occurrences
		    if (value.equals(prevValue)) {
			// This should never happen.. Is it legacy code?
			throw new IllegalStateException("For indexId " + key.getIndex() + " and term " + key.getTerm() + " got a duplicate occurrence "
				+ value.toString());
		    }
		    while (value.equals(prevValue) && valuesIt.hasNext()) {
			value = valuesIt.next();
		    }
		    if (value.equals(prevValue) && !valuesIt.hasNext()) {
			last = true;
		    }
		} else {
		    last = true;
		}
		if (last) {
		    // This is the last occurrence: write out the remaining
		    // positions
		    writerDocValue.setDocument(prevValue.getV1());
		    if (writerDocValue.getDocument() < 0) {
			throw new IllegalStateException("Negative DocID. Key:" + key + "\nprevValue:" + prevValue + "\nValue:" + value + "\nwriterDocValue:"
				+ writerDocValue);
		    }
		    context.write(writerKey, writerDocValue);

		    writerDocValue.clearOccerrences();
		    value = null;
		}
	    }
	}
	termKeysProcessed++;
    }
}