TrainNewsGroups.java

package mia.classifier.ch14;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
import org.apache.mahout.classifier.sgd.L1;
import org.apache.mahout.classifier.sgd.OnlineLogisticRegression;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.vectorizer.encoders.ConstantValueEncoder;
import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder;
import org.apache.mahout.vectorizer.encoders.StaticWordValueEncoder;
import org.apache.mahout.vectorizer.encoders.Dictionary;

import com.google.common.collect.ConcurrentHashMultiset;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multiset;

public class TrainNewsGroups {
	private static final int FEATURES = 10000;
	private static Multiset<String> overallCounts;
	
	public static void main(String[] args) {
		File base = new File(args[0]);
		overallCounts = HashMultiset.create();

		Map<String, Set<Integer>> traceDictionary = new TreeMap<String, Set<Integer>>();
		FeatureVectorEncoder encoder = new StaticWordValueEncoder("body");
		encoder.setProbes(2);
		encoder.setTraceDictionary(traceDictionary);
		FeatureVectorEncoder bias = new ConstantValueEncoder("Intercept");
		bias.setTraceDictionary(traceDictionary);
		FeatureVectorEncoder lines = new ConstantValueEncoder("Lines");
		lines.setTraceDictionary(traceDictionary);
		Dictionary newsGroups = new Dictionary();
		
		OnlineLogisticRegression learningAlgorithm = 
		    new OnlineLogisticRegression(
		          20, FEATURES, new L1())
		        .alpha(1).stepOffset(1000)
		        .decayExponent(0.9) 
		        .lambda(3.0e-5)
		        .learningRate(20);
		
		List<File> files = new ArrayList<File>();
		for (File newsgroup : base.listFiles()) {
		  newsGroups.intern(newsgroup.getName());
		  files.addAll(Arrays.asList(newsgroup.listFiles()));
		}

		Collections.shuffle(files);
		System.out.printf("%d training files\n", files.size());
		
		double averageLL = 0.0;
		double averageCorrect = 0.0;
		double averageLineCount = 0.0;
		int k = 0;
		double step = 0.0;
		int[] bumps = new int[]{1, 2, 5};
		double lineCount = 0;
		
		Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);
		
/*		for (File file : files) {
			  BufferedReader reader = new BufferedReader(new FileReader(file));
			  String ng = file.getParentFile().getName();     
			  int actual = newsGroups.intern(ng);
			  Multiset<String> words = ConcurrentHashMultiset.create();
			  
			  String line = reader.readLine();
			  while (line != null && line.length() > 0) {
			    if (line.startsWith("Lines:")) {              
			      String count = Iterables.get(onColon.split(line), 1);
			      try {
			        lineCount = Integer.parseInt(count);
			        averageLineCount += (lineCount - averageLineCount) 
			            / Math.min(k + 1, 1000);
			      } catch (NumberFormatException e) {
			        lineCount = averageLineCount;
			      }
			    }
			    boolean countHeader = (
			        line.startsWith("From:") || line.startsWith("Subject:")||
			        line.startsWith("Keywords:")|| line.startsWith("Summary:"));
			    do {
			      StringReader in = new StringReader(line);
			      if (countHeader) {
			        countWords(analyzer, words, in);    
			      }
			      line = reader.readLine();
			    } while (line.startsWith(" "));
			  }
			  countWords(analyzer, words, reader);      
			  reader.close();
			}
		
		Vector v = new RandomAccessSparseVector(FEATURES);
		bias.addToVector(null, 1, v);
		lines.addToVector(null, lineCount / 30, v);
		logLines.addToVector(null, Math.log(lineCount + 1), v);
		for (String word : words.elementSet()) {
		  encoder.addToVector(word, Math.log(1 + words.count(word)), v);
		}
		*/
		
		/*double mu = Math.min(k + 1, 200);
		double ll = learningAlgorithm.logLikelihood(actual, v);  #1
		averageLL = averageLL + (ll - averageLL) / mu;

		Vector p = new DenseVector(20);
		learningAlgorithm.classifyFull(p, v);
		int estimated = p.maxValueIndex();

		int correct = (estimated == actual? 1 : 0);
		averageCorrect = averageCorrect + (correct - averageCorrect) / mu;*/
		
/*		learningAlgorithm.train(actual, v);
		k++;
		int bump = bumps[(int) Math.floor(step) % bumps.length];
		int scale = (int) Math.pow(10, Math.floor(step / bumps.length));
		if (k % (bump * scale) == 0) {
		  step += 0.25;
		  System.out.printf("%10d %10.3f %10.3f %10.2f %s %s\n",
		       k, ll, averageLL, averageCorrect * 100, ng, 
		       newsGroups.values().get(estimated));
		}
		learningAlgorithm.close();*/
		
	}

	private static void countWords(Analyzer analyzer, Collection<String> words, Reader in) throws IOException {
	    TokenStream ts = analyzer.tokenStream("text", in);
	    ts.addAttribute(CharTermAttribute.class);
	    while (ts.incrementToken()) {
	      String s = ts.getAttribute(CharTermAttribute.class).toString();
	      words.add(s);
	    }
	    /*overallCounts.addAll(words);*/
	  }
}