MapReduceTest.java

/*
 * JBoss, Home of Professional Open Source.
 * Copyright 2012, Red Hat, Inc., and individual contributors
 * as indicated by the @author tags. See the copyright.txt file in the
 * distribution for a full listing of individual contributors.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */

package org.jboss.test.capedwarf.cluster.test;

import java.util.Arrays;
import java.util.List;

import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.tools.mapreduce.Counter;
import com.google.appengine.tools.mapreduce.Counters;
import com.google.appengine.tools.mapreduce.KeyValue;
import com.google.appengine.tools.mapreduce.MapReduceJob;
import com.google.appengine.tools.mapreduce.MapReduceResult;
import com.google.appengine.tools.mapreduce.MapReduceSpecification;
import com.google.appengine.tools.mapreduce.Marshallers;
import com.google.appengine.tools.mapreduce.inputs.ConsecutiveLongInput;
import com.google.appengine.tools.mapreduce.inputs.DatastoreInput;
import com.google.appengine.tools.mapreduce.outputs.InMemoryOutput;
import com.google.appengine.tools.mapreduce.outputs.NoOutput;
import com.google.appengine.tools.mapreduce.reducers.NoReducer;
import com.google.appengine.tools.pipeline.JobInfo;
import org.jboss.arquillian.container.test.api.Deployment;
import org.jboss.arquillian.container.test.api.OperateOnDeployment;
import org.jboss.arquillian.container.test.api.TargetsContainer;
import org.jboss.arquillian.junit.Arquillian;
import org.jboss.arquillian.junit.InSequence;
import org.jboss.shrinkwrap.api.spec.WebArchive;
import org.jboss.test.capedwarf.common.support.JBoss;
import org.jboss.test.capedwarf.testsuite.LibUtils;
import org.jboss.test.capedwarf.testsuite.mapreduce.support.CountMapper;
import org.jboss.test.capedwarf.testsuite.mapreduce.support.CountReducer;
import org.jboss.test.capedwarf.testsuite.mapreduce.test.MapReduceTestBase;
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;

/**
 * MapReduce cluster tests.
 *
 * @author <a href="mailto:ales.justin@jboss.org">Ales Justin</a>
 */
@RunWith(Arquillian.class)
@Category(JBoss.class)
public class MapReduceTest extends MapReduceTestBase {
    private List<String> payloads = Arrays.asList("capedwarf", "jboss", "redhat");

    public MapReduceTest() {
        ignoreTearDown = true;
    }

    @Deployment(name = "dep1") @TargetsContainer("container-1")
    public static WebArchive getDeploymentA() {
        return getModuleDeployment();
    }

    @Deployment(name = "dep2") @TargetsContainer("container-2")
    public static WebArchive getDeploymentB() {
        return getModuleDeployment();
    }

    protected static WebArchive getModuleDeployment() {
        LibUtils.applyTempModule("cluster-tests");
        try {
            return getDefaultDeployment().addPackage(MyEntityCreator.class.getPackage())
                    .addClass(CountReducer.class)
                    .addClass(CountMapper.class);
        } finally {
            LibUtils.applyTempModule(null);
        }
    }

    @SuppressWarnings("unchecked")
    @Test
    @OperateOnDeployment("dep1") @InSequence(10)
    public void putDataInA() throws Exception {
        int shardCount = 1;

        MapReduceSpecification.Builder builder = new MapReduceSpecification.Builder();
        builder.setJobName("Create MapReduce entities");
        builder.setInput(new ConsecutiveLongInput(0, payloads.size() * (long) shardCount, shardCount));
        builder.setMapper(new MyEntityCreator("MapReduceTest", payloads));
        builder.setKeyMarshaller(Marshallers.getVoidMarshaller());
        builder.setValueMarshaller(Marshallers.getVoidMarshaller());
        builder.setReducer(NoReducer.create());
        builder.setOutput(new NoOutput());

        final String createHandle = MapReduceJob.start(builder.build(), getSettings());

        Entity entity = new Entity("CMR", 1L);
        entity.setProperty("handle", createHandle);
        createDatastoreService().put(entity);

        sync();
    }

    @Test @OperateOnDeployment("dep2") @InSequence(20)
    public void waitForDataInB() throws Exception {
        DatastoreService service = createDatastoreService();
        Entity entity = service.get(KeyFactory.createKey("CMR", 1L));
        String createHandle = entity.getProperty("handle").toString();
        service.delete(entity.getKey()); // cleanup

        JobInfo createJI = waitToFinish("CREATE", createHandle);
        Object create = createJI.getOutput();
        log.severe("----- Create: " + create);
    }

    @SuppressWarnings("unchecked")
    @Test
    @OperateOnDeployment("dep2") @InSequence(30)
    public void countDataInB() throws Exception {
        int mapShardCount = 1;

        MapReduceSpecification.Builder builder = new MapReduceSpecification.Builder();
        builder.setJobName("MapReduceTest stats");
        builder.setInput(new DatastoreInput("MapReduceTest", mapShardCount));
        builder.setMapper(new CountMapper());
        builder.setKeyMarshaller(Marshallers.getStringMarshaller());
        builder.setValueMarshaller(Marshallers.getLongMarshaller());
        builder.setReducer(new CountReducer());
        builder.setOutput(new InMemoryOutput<KeyValue<String, Long>>());

        final String countHandle = MapReduceJob.start(builder.build(), getSettings());

        Entity entity = new Entity("CMR", 2);
        entity.setProperty("handle", countHandle);
        createDatastoreService().put(entity);

        sync();
    }

    @Test
    @OperateOnDeployment("dep1") @InSequence(40)
    public void waitForCountInA() throws Exception {
        Entity entity = createDatastoreService().get(KeyFactory.createKey("CMR", 2));
        String countHandle = entity.getProperty("handle").toString();

        JobInfo countJI = waitToFinish("COUNT", countHandle);
        Object count = countJI.getOutput();
        log.severe("----- Count: " + count);
    }

    @Test
    @OperateOnDeployment("dep2") @InSequence(50)
    public void verifyInB() throws Exception {
        DatastoreService service = createDatastoreService();
        Entity entity = service.get(KeyFactory.createKey("CMR", 2));
        String countHandle = entity.getProperty("handle").toString();
        service.delete(entity.getKey()); // cleanup

        JobInfo jobInfo = getJobInfo("VERIFY", countHandle);
        Assert.assertTrue("Not finished: " + toInfo(jobInfo), jobInfo.getJobState() == JobInfo.State.COMPLETED_SUCCESSFULLY);

        Object count = jobInfo.getOutput();
        Assert.assertTrue(count instanceof MapReduceResult);
        MapReduceResult result = MapReduceResult.class.cast(count);
        int[] chars = toChars(payloads);
        Counters counters = result.getCounters();
        for (int i = 0; i < chars.length; i++) {
            char ch = (char) ('a' + i);
            Counter c = counters.getCounter(CountMapper.toKey(ch));
            Assert.assertEquals(String.format("Invalid count for '%s'.", ch), chars[i], c.getValue());
        }
    }

    @Test @OperateOnDeployment("dep1") @InSequence(60)
    public void cleanupOnA() throws Exception {
        cleanup();
    }

    @Test @OperateOnDeployment("dep1") @InSequence(70)
    public void waitToFinishA() throws Exception {
        sync();
    }

    @Test @OperateOnDeployment("dep2") @InSequence(80)
    public void waitToFinishB() throws Exception {
        sync();
    }

    protected static int[] toChars(List<String> payloads) {
        int[] chars = new int['z' - 'a' + 1];
        for (String payload : payloads) {
            for (int i = 0; i < payload.length(); i++) {
                chars[payload.charAt(i) - 'a']++;
            }
        }
        return chars;
    }
}