/* * Copyright 2014 Rackspace * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.rackspacecloud.blueflood.CloudFilesBackfiller.service; import com.rackspacecloud.blueflood.CloudFilesBackfiller.rollup.handlers.FileHandler; import com.rackspacecloud.blueflood.CloudFilesBackfiller.rollup.handlers.RollupGenerator; import com.rackspacecloud.blueflood.service.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FilenameFilter; import java.util.Arrays; import java.util.Comparator; import java.util.concurrent.CountDownLatch; public class OutOFBandRollup { private static final Logger log = LoggerFactory.getLogger(OutOFBandRollup.class); private static final FileHandler fileHandler = new FileHandler(); private static Thread monitoringThread; private static Thread rollupGeneratorThread; private static boolean running; public static void main(String args[]) { final File file = new File(Configuration.getInstance().getStringProperty(BackFillerConfig.DOWNLOAD_DIR)); if(!file.exists() || !file.isDirectory()) { log.error("Download directory not properly configured"); System.exit(-1); } File rollupDir = new File(Configuration.getInstance().getStringProperty(BackFillerConfig.ROLLUP_DIR)); rollupDir.mkdirs(); running = true; monitoringThread = new Thread(){ public void run() { while(running) { log.info("Scanning for downloaded files"); FilenameFilter filter = new FilenameFilter() { public boolean accept(File dir, String name) { return name.endsWith(".json.gz"); } }; File[] cloudFiles = file.listFiles(filter); Comparator<File> fileComparator = new Comparator<File>() { @Override public int compare(File f1, File f2) { return (int) (getTimeStamp(f1) - getTimeStamp(f2)); } private long getTimeStamp(File file) { String fileName = file.getName(); // 20140226_1393442533000.json.gz String dateAndTs = fileName.split("\\.", 2)[0].trim(); // 20140226_1393442533000 String tsCreated = dateAndTs.split("_")[1].trim(); // 1393442533000 return Long.parseLong(tsCreated); } }; // Throttled the cloud files being parsed if (cloudFiles.length != 0 && RollupGenerator.rollupExecutors.getQueue().size() < 10000) { Arrays.sort(cloudFiles, fileComparator); CountDownLatch latch = new CountDownLatch(cloudFiles.length); fileHandler.setCountDown(latch); for(File file : cloudFiles) { fileHandler.fileReceived(file); } try { log.info("Waiting for latch to be released"); latch.await(); } catch (InterruptedException e) { } } else { log.info("Sleeping for some time because cloud files have not yet cleared/rollup queue size is way too high"); try { Thread.sleep(10000); } catch (InterruptedException e) { running = false; } } } } }; monitoringThread.start(); rollupGeneratorThread = new Thread(new RollupGenerator()); rollupGeneratorThread.start(); } public static Thread getMonitoringThread() { return monitoringThread; } public static Thread getRollupGeneratorThread() { return rollupGeneratorThread; } }