/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.InterSedes;
import org.apache.pig.data.InterSedesFactory;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.util.TupleFormat;
import org.junit.Test;
public class TestBinInterSedes {
@Test
public void testTupleWriteRead1() throws IOException {
try {
//create a tuple with columns of different type
Tuple tuplein = TupleFactory.getInstance().newTuple(7);
tuplein.set(0, 12);
Map<String, String> map = new HashMap<String, String>();
map.put("pig", "scalability");
tuplein.set(1, map);
tuplein.set(2, null);
tuplein.set(3, 12L);
tuplein.set(4, 1.2F);
Tuple innerTuple = TupleFactory.getInstance().newTuple(1);
innerTuple.set(0, "innerTuple");
tuplein.set(5, innerTuple);
DataBag bag = BagFactory.getInstance().newDefaultBag();
bag.add(innerTuple);
tuplein.set(6, bag);
testTupleSedes(tuplein);
assertEquals(
"(12,[pig#scalability],,12,1.2,(innerTuple),{(innerTuple)})",
TupleFormat.format(tuplein));
} catch (ExecException e) {
e.printStackTrace();
fail();
}
}
/**
* test sedes of int of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadIntDiffSizes() throws IOException {
try {
//create a tuple with integer columns of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
tuple.append(new Integer(0)); //boolean rep
tuple.append(new Integer(1)); //boolean rep
tuple.append(new Integer(125)); //fits into byte
tuple.append(new Integer(1024)); //fits into short
tuple.append(new Integer(1024*1024*1024)); //fits into int (=~ 2 ^30)
testTupleSedes(tuple);
} catch (ExecException e) {
e.printStackTrace();
fail();
}
}
/**
* test sedes of bytearray, string of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadByteArrStringDiffSizes() throws IOException {
try {
// tuple with ByteArray and strings of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
byte [] tinyBA = new byte[10];
byte [] smallBA = new byte[1000];
byte [] largeBytearray = new byte[80000];
//init large bytearray with non 0 values, its going to be used as
//string as well
for(int i=0; i< largeBytearray.length; i++){
largeBytearray[i] = '1';
}
tuple.append(new DataByteArray(tinyBA));
tuple.append(new DataByteArray(smallBA));
tuple.append(new DataByteArray(largeBytearray));
testTupleSedes(tuple);
// add strings of different sizes
tuple = TupleFactory.getInstance().newTuple();
tuple.append(new String(""));
tuple.append(new String("x"));
//string larger than 32k
tuple.append(new String(largeBytearray));
testTupleSedes(tuple);
} catch (ExecException e) {
e.printStackTrace();
fail();
}
}
/**
* test sedes with bags of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadBagDiffSizes() throws IOException {
try {
// tuple with ByteArray and strings of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
DataBag tinyBag = createBag(10);
DataBag smallBag = createBag(1000);
DataBag largeBag = createBag(100*1024);
tuple.append(tinyBag);
tuple.append(smallBag);
tuple.append(largeBag);
testTupleSedes(tuple);
} catch (ExecException e) {
e.printStackTrace();
fail();
}
}
/**
* create bag having given number of tuples
* @param size
* @return
*/
private DataBag createBag(int size) {
Tuple innerTuple = TupleFactory.getInstance().newTuple();
innerTuple.append(Integer.valueOf(1));
DataBag bag = BagFactory.getInstance().newDefaultBag();
for(int i=0; i<size; i++){
bag.add(innerTuple);
}
return bag;
}
/**
* test sedes tuple of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadDiffSizes() throws IOException {
try {
// tuple with ByteArray and strings of different sizes
Tuple smallTuple = createTupleWithManyCols(1000);
testTupleSedes(smallTuple);
Tuple largeTuple = createTupleWithManyCols(100*1000);
testTupleSedes(largeTuple);
} catch (ExecException e) {
e.printStackTrace();
fail();
}
}
private Tuple createTupleWithManyCols(int size) {
Tuple t = TupleFactory.getInstance().newTuple(size);
Integer col = Integer.valueOf(1);
for(int i=0; i<size; i++){
t.append(col);
}
return t;
}
/**
* test sedes with maps of diff sizes
* @throws IOException
*/
@Test
public void testTupleWriteReadMapDiffSizes() throws IOException {
try {
// tuple with ByteArray and strings of different sizes
Tuple tuple = TupleFactory.getInstance().newTuple();
Map<String, Object> tinyMap = createMap(10);
Map<String, Object> smallMap = createMap(1000);
Map<String, Object> largeMap = createMap(100*1024);
tuple.append(tinyMap);
tuple.append(smallMap);
tuple.append(largeMap);
testTupleSedes(tuple);
} catch (ExecException e) {
e.printStackTrace();
fail();
}
}
private Map<String, Object> createMap(int size) {
Map<String,Object> map = new HashMap<String, Object>(size);
String key = String.valueOf('A');
Integer val = Integer.valueOf(1);
for(int i=0; i<size; i++){
map.put(key, val);
}
return map;
}
/**
* Write the serialized tuple to DataOutputStream and get deserialized tuple
* compare the given tuple and deserialized tuple to make sure they are
* equivalent
* @param tuple
* @throws IOException
*/
private void testTupleSedes(Tuple tuple) throws IOException {
InterSedes sedes = InterSedesFactory.getInterSedesInstance();
//write the tuple into a DataOutputStream on bytearray
ByteArrayOutputStream bout = new ByteArrayOutputStream(10*1024*1024);// 10 MB
DataOutputStream out = new DataOutputStream(bout);
sedes.writeDatum(out, tuple);
out.flush();
//read tuple back
ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
DataInputStream in = new DataInputStream(bin);
Tuple tupleout = (Tuple)sedes.readDatum(in);
assertEquals(" Tuple before and after serialization are same ",
tuple, tupleout);
}
}