/* * Copyright 2012, Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.facebook.LinkBench; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Random; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import junit.framework.TestCase; import org.junit.Test; import org.junit.experimental.categories.Category; import com.facebook.LinkBench.generators.DataGenerator; import com.facebook.LinkBench.generators.MotifDataGenerator; import com.facebook.LinkBench.generators.UniformDataGenerator; import com.facebook.LinkBench.testtypes.SlowTest; @Category(SlowTest.class) public class TestDataGen extends TestCase { public static void printByteGrid(byte[] data) { for (int i = 0; i < data.length; i += 32) { for (int j = i; j < Math.min(i + 32, data.length); j++) { System.err.format("%3d ", data[j]); } System.err.println(); } } /** * Test how quickly uniform data generator can generate patterns */ @Test public void testTimingUniform() { System.err.println("Testing uniform generator"); System.err.println("========================="); DataGenFactory fact = new DataGenFactory() { public DataGenerator make(double param) { UniformDataGenerator gen = new UniformDataGenerator(); gen.init(0, 8); return gen; } }; testTiming(fact, 128); testTiming(fact, 1024); } /** * Test how quickly motif data generator can generate patterns to make * sure its not */ @Test public void testTimingMotif() { System.err.println("Testing motif generator"); System.err.println("========================="); DataGenFactory fact = new DataGenFactory() { public DataGenerator make(double param) { MotifDataGenerator gen = new MotifDataGenerator(); gen.init(0, 8, param); return gen; } }; testTiming(fact, 128); testTiming(fact, 1024); } private void testTiming(DataGenFactory fact, int bufSize) { byte buf[] = new byte[bufSize]; Random rng = new Random(); int trials = 200000; double params[] = new double[] {0.0, 0.25, 0.5, 0.75, 1.0}; long times_ns[] = new long[params.length]; for (int i = 0; i < params.length; i++) { double param = params[i]; // Warm up doTest(fact, buf, rng, trials, param); // Make sure hotspot will have compiled try { Thread.sleep(50); } catch (InterruptedException e) { e.printStackTrace(); } long timeTaken = doTest(fact, buf, rng, trials, param); times_ns[i] = timeTaken; } for (int i = 0; i < params.length; i++) { double trialTime = times_ns[i] / (double) trials; double byteTime = trialTime / buf.length; System.err.format("uniqueness = %.3f, time for %d byte buffer = %.1f ns, time per byte = %.1fns\n", params[i], buf.length, trialTime, byteTime); } } private static interface DataGenFactory { public abstract DataGenerator make(double param); } private long doTest(DataGenFactory fact, byte[] buf, Random rng, int trials, double param) { DataGenerator gen = fact.make(param); long start = System.nanoTime(); for (int j = 0; j < trials; j++) { gen.fill(rng, buf); } long end = System.nanoTime(); long timeTaken = end - start; return timeTaken; } /** * Exercise the motif data generator and print the output. * * Currently difficult to automatically verify output. */ @Test public void testMotif() { MotifDataGenerator gen = new MotifDataGenerator(); System.err.println("uniqueness 0.25"); gen.init(0, 8, 0.25); byte data[] = gen.fill(new Random(), new byte[64]); printByteGrid(data); System.err.println("uniqueness 0.0"); gen.init(0, 8, 0.0); data = gen.fill(new Random(), new byte[64]); printByteGrid(data); System.err.println("uniqueness 0.05"); gen.init(0, 8, 0.05); data = gen.fill(new Random(), new byte[64]); printByteGrid(data); System.err.println("uniqueness 1.0"); gen.init(0, 8, 1.0); data = gen.fill(new Random(), new byte[64]); printByteGrid(data); } /** * Estimate the compressibility of randomly generated data by * compressing a long stream of the data * @throws IOException */ @Test public void testCompressibility() throws IOException { MotifDataGenerator gen = new MotifDataGenerator(); gen.init(0, 255, 0.5); System.err.println("\nUniqueness=0.5 Range=255\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); gen.init(0, 127, 0.5); System.err.println("\nUniqueness=0.5 Range=127\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); gen.init(0, 255, 0.0); System.err.println("\nUniqueness=0.0 Range=255\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); gen.init(0, 255, 1.0); System.err.println("\nUniqueness=1.0 Range=255\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); gen.init(0, 255, 1.0); gen.init(0, 127, 1.0); System.err.println("\nUniqueness=1.0 Range=127\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); gen.init(0, 1, 1.0); System.err.println("\nUniqueness=1.0 Range=1\n==============="); testCompressibility(gen, 1024, 10000); testCompressibility(gen, 64, 1); } private void testCompressibility(MotifDataGenerator gen, int blockSize, int blocks) throws IOException { long seed = System.nanoTime(); System.err.println("seed = " + seed); Random rng = new Random(seed); ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); Deflater def = new Deflater(Deflater.BEST_COMPRESSION); DeflaterOutputStream gzipOut = new DeflaterOutputStream(byteOut, def); byte block[] = new byte[blockSize]; for (int i = 0; i < blocks; i++) { gen.fill(rng, block); gzipOut.write(block); } gzipOut.close(); byte compressed[] = byteOut.toByteArray(); int origLen = blockSize * blocks; System.err.format("%dx%d blocks. Compressed %d bytes to %d: %.2f. Bound: %.2f\n", blocks, blockSize, origLen, compressed.length, compressed.length / (double) origLen, gen.estMaxCompression()); } }