Tools.java

package logdruid.util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import logdruid.data.DateFormat;
import logdruid.data.Preferences;
import logdruid.data.Repository;
import logdruid.data.Source;
import logdruid.data.mine.ChartData;
import logdruid.data.mine.FileRecord;
import logdruid.data.record.Recording;

public class Tools {
	private static Logger logger = Logger.getLogger(Tools.class.getName());

	public Tools() {
		// TODO Auto-generated constructor stub
	}

	public static boolean generateFlatFiles(Repository repo, File folderLocation) {

		// PatternCache patternCache = new PatternCache();
		FileReader flstr = null;
		BufferedReader buf1st = null;
		Map<Recording, String> recMatch = new HashMap<Recording, String>();
		Matcher matcher;
		Matcher matcher2;
		BufferedWriter bufWriter = null;
		if (repo.getBaseSourcePath() == null)
			return false;
		File folder = new File(repo.getBaseSourcePath());
		if (repo != null && repo.getBaseSourcePath() != null) {
			ChartData cd = DataMiner.gatherSourceData(repo, true);
			ArrayList sources = repo.getSources();
			Iterator sourceArrayListIte = sources.iterator();
			while (sourceArrayListIte.hasNext()) {
				cd.sourceArrayList = repo.getSources();
				Iterator<Source> sourceIterator = cd.sourceArrayList.iterator();
				Source src = (Source) sourceArrayListIte.next();
				Map<String, ArrayList<FileRecord>> hm = cd.getGroupFilesMap(src);
				logger.info("Source: " + src.getSourceName());
				if (hm != null && hm.entrySet() != null) {
					Iterator it = hm.entrySet().iterator();
					while (it.hasNext()) {
						final Map.Entry pairs = (Map.Entry) it.next();
						// removing file separators so that files are stored
						// inside one folder and to prevent no such directory
						// error
						logger.info("original file string: " + pairs.getKey().toString());
						String noSeparatorString = pairs.getKey().toString().replace("/", "-");
						logger.info("updated file string: " + noSeparatorString);
						File oneFile = new File(folderLocation.getAbsolutePath() + File.separator + noSeparatorString);
						logger.info("complete file string: " + oneFile.getAbsolutePath());
						FileWriter fileWriter = null;
						try {
							fileWriter = new FileWriter(oneFile);
						} catch (IOException e2) {
							// TODO Auto-generated catch block
							e2.printStackTrace();
						}
						if (fileWriter != null) {
							bufWriter = new BufferedWriter(fileWriter);
							logger.info("Source group: " + pairs.getKey());
							ArrayList<FileRecord> grouFile = (ArrayList<FileRecord>) pairs.getValue();
							// return DataMiner.mine((String) pairs.getKey(),
							// (ArrayList<String>) pairs.getValue(), repo,
							// source,
							// repo.isStats(), repo.isTimings());
							Iterator<FileRecord> fileArrayListIterator = grouFile.iterator();
							while (fileArrayListIterator.hasNext()) {
								final FileRecord fileName = fileArrayListIterator.next();
								try {
									flstr = new FileReader(new File(fileName.getCompletePath()));
								} catch (FileNotFoundException e1) {
									// TODO Auto-generated catch block
									e1.printStackTrace();
								}
								if (flstr != null) {
									buf1st = new BufferedReader(flstr);
									String line;
									logger.info("wrote file:" + fileName.getCompletePath());
									// try {
									try {
										while ((line = buf1st.readLine()) != null) {
											bufWriter.write(line);
											bufWriter.newLine();
										}
									} catch (IOException e) {
										// TODO Auto-generated catch block
										e.printStackTrace();
									}
								}
								
								try {
									buf1st.close();
								} catch (IOException e) {
									// TODO Auto-generated catch block
									e.printStackTrace();
								}
							}
						}
						try {
							bufWriter.close();
						} catch (IOException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}
						
					}
				}
			}

		}
		return true;

	}

	// ordering files by date of lines
	public static ArrayList<FileRecord> orderFiles(ArrayList<FileRecord> groupfiles, Source src) {
		// String test = Preferences.getPreference("ThreadPool_fileOrder");
		FileReader flstr = null;
		BufferedReader buf1st = null;
		Map<Date, FileRecord> tempDateFile = new HashMap<Date, FileRecord>();
		ArrayList<FileRecord> finaList = new ArrayList<FileRecord>();
		DateFormat df = src.getDateFormat();
		if (df != null) {
			Pattern srcDatePattern = Pattern.compile(df.getPattern() + ".*");
			logger.debug(df.getPattern());
			java.text.DateFormat fastDateFormat = ThreadLocalDateFormatMap.getInstance().createSimpleDateFormat(df.getDateFormat());
			if (groupfiles != null) {
				if (df.getDateFormat() != "") {
					for (FileRecord fileRecord : groupfiles) {
						try {
							if (logger.isDebugEnabled()) {
								logger.debug("looking for first date in file: " + (String) fileRecord.getCompletePath().toString());
							}
							flstr = new FileReader(fileRecord.getCompletePath());
						} catch (FileNotFoundException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}
						buf1st = new BufferedReader(flstr);
						String line;
						int lineCount = 0;
						try {
							// recMatch = getRegexp(repo, source);
							// checking lines until a date is found
							while ((line = buf1st.readLine()) != null && lineCount < 1000) {
								Matcher matcher = srcDatePattern.matcher(line);
								if (matcher.find()) { // date found, now parsing
														// to date
									try {
										Date date1 = fastDateFormat.parse(matcher.group(0));
										tempDateFile.put(date1, fileRecord);
										break;
									} catch (ParseException e) {
										// TODO Auto-generated catch block
										e.printStackTrace();
										logger.info(df.getPattern());
									}
								}
							}
						} catch (IOException e1) {
							// TODO Auto-generated catch block
							e1.printStackTrace();
						} finally {
							try {
								buf1st.close();
							} catch (IOException e) {
								// TODO Auto-generated catch block
								e.printStackTrace();
							}
						}

					}
				} else {
					return groupfiles;
				}
			}

			Iterator<Date> ite = tempDateFile.keySet().iterator();

			while (ite.hasNext()) {
				Date date = (Date) ite.next();
				logger.debug("BEFORE ORDERING - date: " + date + "File: " + ((FileRecord) tempDateFile.get(date)).getCompletePath());

			}
			List<Date> test1 = new ArrayList<Date>();

			// sorting list of dates
			test1.addAll(tempDateFile.keySet());
			Collections.sort(test1, new Comparator<Date>() {
				public int compare(Date o1, Date o2) {
					return Long.compare(o1.getTime(), o2.getTime());
				}
			});
			Iterator<Date> ite2 = test1.iterator();

			while (ite2.hasNext()) {
				Date date = (Date) ite2.next();
				finaList.add(tempDateFile.get(date));
				logger.debug("ORDERED - date: " + date + "File: " + ((FileRecord) tempDateFile.get(date)).getCompletePath());

			}

			return finaList;
		} else
			return groupfiles;

	}

}