package org.openstack.atlas.logs.itest;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import org.joda.time.DateTime;
import org.json.simple.parser.ParseException;
import org.openstack.atlas.config.HadoopLogsConfigs;
import org.openstack.atlas.logs.hadoop.comparators.LoadBalancerIdAndNameComparator;
import org.openstack.atlas.service.domain.pojos.LoadBalancerIdAndName;
import org.openstack.atlas.logs.hadoop.util.HdfsUtils;
import org.openstack.atlas.logs.hadoop.util.LogFileNameBuilder;
import org.openstack.atlas.util.debug.Debug;
import org.openstack.atlas.util.itest.hibernate.HibernateDbConf;
import org.openstack.atlas.util.itest.hibernate.HuApp;
import org.openstack.atlas.util.staticutils.StaticFileUtils;
import org.openstack.atlas.logs.hadoop.util.StaticLogUtils;
import org.openstack.atlas.util.staticutils.StaticDateTimeUtils;
public class CacheFakerMain {
public static final Random rnd = new Random();
public static final int BUFFSIZE = 1024 * 32;
private static Set<String> mkDirSet = new HashSet<String>();
public static void writeZipFile(String cacheDir, LoadBalancerIdAndName lb, int fileHour, boolean buildZips) throws FileNotFoundException, UnsupportedEncodingException, IOException {
int accountId = lb.getAccountId();
int loadbalancerId = lb.getLoadbalancerId();
String lbName = lb.getName();
String zipName = LogFileNameBuilder.getZipFileName(loadbalancerId, fileHour);
String zipContentsName = LogFileNameBuilder.getZipContentsName(loadbalancerId, fileHour);
String lidStr = Integer.toString(loadbalancerId);
String aidStr = Integer.toString(accountId);
String fileHourStr = Integer.toString(fileHour);
String containerName = LogFileNameBuilder.getContainerName(lidStr, lbName, fileHourStr); // CloudFiles container name
String cfName = LogFileNameBuilder.getRemoteFileName(lidStr, lbName, fileHourStr); // CloudFiles name
String zipDir = StaticFileUtils.mergePathString(cacheDir, fileHourStr, aidStr);
// Attempt to make the directory
File zipDirFile = new File(zipDir);
if (!mkDirSet.contains(zipDirFile.getAbsolutePath())) {
System.out.printf("Creating directory %s ", zipDir);
System.out.printf("%s\n", zipDirFile.mkdirs());
mkDirSet.add(zipDirFile.getAbsolutePath());
}
if (!buildZips) {
return; // The user just wants directorys
}
String zipFilePath = StaticFileUtils.mergePathString(zipDir, zipName);
System.out.printf("Opening %s for writing\n", zipFilePath);
OutputStream os = StaticFileUtils.openOutputFile(zipFilePath);
ZipOutputStream zos = new ZipOutputStream(os);
zos.setComment(String.format("Fake log file generated by CacheFakerMain"));
zos.putNextEntry(new ZipEntry(zipContentsName));
String fmt = "This file was generated by CachFakerMain\ndate %s\nContainer %s\nremoteFile %s\naid %s\nlid %s\n";
DateTime now = StaticDateTimeUtils.nowDateTime(true);
String dateStr = StaticDateTimeUtils.sqlDateTimeFormat.print(now);
String msg = String.format(fmt, dateStr, containerName, cfName, aidStr, lidStr);
byte[] msgBytes = msg.getBytes("utf-8");
zos.write(msgBytes);
zos.closeEntry();
zos.finish();
os.close();
}
public static void main(String[] args) throws ParseException, UnsupportedEncodingException, FileNotFoundException, IOException {
Map<String, String> kwArgs = CommonItestStatic.argMapper(args);
String[] nonKeywordArgs = CommonItestStatic.stripKwArgs(args);
if (nonKeywordArgs.length < 2) {
System.out.printf("Usage is <configFile> <startHour> [stopHour] [outdir=someDir] ");
System.out.printf("[aid=SomeAid] [lid=SomeLid] [dirsonly=<true|false>\n");
System.out.printf("\n");
System.out.printf("Create fake zip files for testing the cache reuploader\n");
System.out.printf("Use the keyword args to limit zips to only the specified account or loadbalancers\n");
System.out.printf("%s\n", HibernateDbConf.exampleJson);
return;
}
BufferedReader stdin = StaticFileUtils.inputStreamToBufferedReader(System.in, BUFFSIZE);
//System.out.printf("Press enter to continue\n");
//stdin.readLine();
String jsonConfFileName = StaticFileUtils.expandUser(nonKeywordArgs[0]);
long begHour = Long.parseLong(nonKeywordArgs[1]);
long endHour = (nonKeywordArgs.length > 2) ? Long.parseLong(nonKeywordArgs[2]) : begHour;
Integer aid = (kwArgs.containsKey("aid")) ? Integer.parseInt(kwArgs.get("aid")) : null;
Integer lid = (kwArgs.containsKey("lid")) ? Integer.parseInt(kwArgs.get("lid")) : null;
boolean buildZips = true; // If the user only wants directories only then set build to false
if (kwArgs.containsKey("dirsonly")) {
String val = kwArgs.get("dirsonly");
if (val.equalsIgnoreCase("true")) {
buildZips = false;
}
}
HuApp huApp = new HuApp();
HibernateDbConf hConf = HibernateDbConf.newHibernateConf(jsonConfFileName);
System.out.printf("Useing db config %s\n", hConf.toString());
huApp.setDbMap(hConf);
System.out.printf("Reading LoadBalancers from databases\n");
double startTime = Debug.getEpochSeconds();
Map<Integer, LoadBalancerIdAndName> lbMap = CommonItestStatic.getLbIdMap(huApp);
lbMap = CommonItestStatic.filterLbIdMap(lbMap, aid, lid);
double stopTime = Debug.getEpochSeconds();
int nLbsFound = lbMap.size();
double deltaTime = stopTime - startTime;
System.out.printf("Took %f seconds to produce read %d records\n", deltaTime, nLbsFound);
String outdir = HadoopLogsConfigs.getCacheDir();
if (kwArgs.containsKey("outdir")) {
outdir = kwArgs.get("outdir");
}
List<LoadBalancerIdAndName> lbs = new ArrayList<LoadBalancerIdAndName>(lbMap.values());
Collections.sort(lbs, new LoadBalancerIdAndNameComparator());
List<Long> hourKeys = StaticLogUtils.getHourKeysInRange(begHour, endHour);
Collections.sort(hourKeys);
System.out.printf("Generating zip files for hours: ");
for (Long hourKey : hourKeys) {
System.out.printf("%d\n", hourKey);
}
System.out.printf("\n");
System.out.printf("Will be generating zips for:\n");
for (LoadBalancerIdAndName lb : lbs) {
System.out.printf("aid[%8d] lb[%6d] \"%s\"\n", lb.getAccountId(), lb.getLoadbalancerId(), lb.getName());
}
long nZips = (long) hourKeys.size() * (long) lbs.size();
System.out.printf("Are you sure you want to generate %d logs\n", nZips);
if (CommonItestStatic.inputStream(stdin, "Y")) {
System.out.printf("Generating zips:\n");
for (Long hourKey : hourKeys) {
for (LoadBalancerIdAndName lb : lbs) {
if (buildZips) {
System.out.printf("Writing file for aid[%8d] lid[%8d]\n", lb.getAccountId(), lb.getLoadbalancerId());
}
writeZipFile(outdir, lb, hourKey.intValue(), buildZips);
}
}
} else {
System.out.printf("Not generating zips\n");
}
}
}