/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataByteArray;
import org.apache.pig.data.DataType;
import org.apache.pig.data.InternalMap;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.junit.Test;
/**
* This class will exercise the basic Pig data model and members. It tests for proper behavior in
* assigment and comparision, as well as function application.
*/
public class TestDataModel {
@Test
public void testDatum() throws Exception {
// Datum is an empty class, but in the event we
// ever add a method, test it here
}
@Test
public void testTuple() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
int arity = 5;
int[] input1 = { 1, 2, 3, 4, 5 };
String[] input2 = { "1", "2", "3", "4", "5" };
String[] input3 = { "1", "2", "3", "4", "5", "6" };
// validate construction and equality
Tuple f1 = Util.loadFlatTuple(tf.newTuple(arity), input1);
Tuple f2 = Util.loadFlatTuple(tf.newTuple(arity), input1);
Tuple f3 = tf.newTuple(arity);
assertEquals(arity, f1.size());
assertEquals(f1, f2);
// invalid equality
f2 = Util.loadTuple(tf.newTuple(input3.length), input3);
assertFalse(f1.equals(f2));
// copy equality
/*
* f2.copyFrom(f1);
* assertTrue(f1.equals(f2));
*/
// append function and equality
int[] input4 = { 1, 2, 3 };
int[] input5 = { 4, 5 };
/*
* f1 = Util.loadFlatTuple(tf.newTuple(input4.length), input4);
* f2 = Util.loadFlatTuple(tf.newTuple(input5.length), input5);
* f3 = Util.loadFlatTuple(tf.newTuple(input1.length), input1);
* f1.appendTuple(f2);
* assertTrue(f3.equals(f1));
*/
// arity then value comparision behavior
f1 = Util.loadFlatTuple(tf.newTuple(input1.length), input1); // 1,2,3,4,5
f2 = Util.loadFlatTuple(tf.newTuple(input4.length), input4); // 1,2,3
assertTrue(f1.compareTo(f2) > 0);
assertFalse(f1.compareTo(f2) < 0);
int[] input6 = { 1, 2, 3, 4, 6 };
f2 = Util.loadFlatTuple(tf.newTuple(input6.length), input6);
assertTrue(f1.compareTo(f2) < 0);
assertFalse(f1.compareTo(f2) > 0);
// delimited export
String expected = "1:2:3:4:5";
f1 = Util.loadFlatTuple(tf.newTuple(input1.length), input1);
assertTrue(expected.equals(f1.toDelimitedString(":")));
// value read / write & marshalling
PipedOutputStream pos = new PipedOutputStream();
DataOutputStream dos = new DataOutputStream(pos);
PipedInputStream pis = new PipedInputStream(pos);
DataInputStream dis = new DataInputStream(pis);
f1.write(dos);
f1.write(dos);
f2.readFields(dis);
assertTrue(f1.equals(f2));
f2.readFields(dis);
assertTrue(f1.equals(f2));
}
@Test
public void testNestTuple() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
int[][] input1 = { { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 },
{ 1, 2, 3, 4, 5 }, { 1, 2, 3, 4, 5 } };
int[][] input2 = { { 1, 2 }, { 1, 2 } };
Tuple n1 = Util.loadNestTuple(tf.newTuple(input1.length), input1);
Tuple n2 = tf.newTuple();
n2 = Util.loadNestTuple(tf.newTuple(input2.length), input2);
}
@Test
public void testReadWrite() throws Exception {
// Create a tuple with every data type in it, and then read and
// write it, both via DataReaderWriter and Tuple.readFields
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = giveMeOneOfEach();
File file = File.createTempFile("Tuple", "put");
FileOutputStream fos = new FileOutputStream(file);
DataOutput out = new DataOutputStream(fos);
t1.write(out);
t1.write(out); // twice in a row on purpose
fos.close();
FileInputStream fis = new FileInputStream(file);
DataInput in = new DataInputStream(fis);
for (int i = 0; i < 2; i++) {
Tuple after = tf.newTuple();
after.readFields(in);
Object o = after.get(0);
assertTrue("isa Tuple", o instanceof Tuple);
Tuple t3 = (Tuple)o;
o = t3.get(0);
assertTrue("isa Integer", o instanceof Integer);
assertEquals(new Integer(3), (Integer)o);
o = t3.get(1);
assertTrue("isa Float", o instanceof Float);
assertEquals(new Float(3.0), (Float)o);
o = after.get(1);
assertTrue("isa Bag", o instanceof DataBag);
DataBag b = (DataBag)o;
Iterator<Tuple> j = b.iterator();
Tuple[] ts = new Tuple[2];
assertTrue("first tuple in bag", j.hasNext());
ts[0] = j.next();
assertTrue("second tuple in bag", j.hasNext());
ts[1] = j.next();
o = ts[0].get(0);
assertTrue("isa Integer", o instanceof Integer);
assertEquals(new Integer(4), (Integer)o);
o = ts[1].get(0);
assertTrue("isa String", o instanceof String);
assertEquals("mary had a little lamb", (String)o);
o = after.get(2);
assertTrue("isa Map", o instanceof Map);
Map<String, Object> m = (Map<String, Object>)o;
assertEquals("world", (String)m.get("hello"));
assertEquals("all", (String)m.get("goodbye"));
assertNull(m.get("fred"));
o = after.get(3);
assertTrue("isa Integer", o instanceof Integer);
Integer ii = (Integer)o;
assertEquals(new Integer(42), ii);
o = after.get(4);
assertTrue("isa Long", o instanceof Long);
Long l = (Long)o;
assertEquals(new Long(5000000000L), l);
o = after.get(5);
assertTrue("isa Float", o instanceof Float);
Float f = (Float)o;
assertEquals(new Float(3.141592654), f);
o = after.get(6);
assertTrue("isa Double", o instanceof Double);
Double d = (Double)o;
assertEquals(new Double(2.99792458e8), d);
o = after.get(7);
assertTrue("isa Boolean", o instanceof Boolean);
Boolean bool = (Boolean)o;
assertTrue(bool);
o = after.get(8);
assertTrue("isa DataByteArray", o instanceof DataByteArray);
DataByteArray ba = (DataByteArray)o;
assertEquals(new DataByteArray("hello"), ba);
o = after.get(9);
assertTrue("isa String", o instanceof String);
String s = (String)o;
assertEquals("goodbye", s);
}
file.delete();
}
@Test
public void testReadWriteInternal() throws Exception {
// Create a tuple with every internal data type in it, and then read and
// write it, both via DataReaderWriter and Tuple.readFields
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = tf.newTuple(1);
InternalMap map = new InternalMap(2);
map.put(new Integer(1), new String("world"));
map.put(new Long(3L), new String("all"));
t1.set(0, map);
File file = File.createTempFile("Tuple", "put");
FileOutputStream fos = new FileOutputStream(file);
DataOutput out = new DataOutputStream(fos);
t1.write(out);
fos.close();
FileInputStream fis = new FileInputStream(file);
DataInput in = new DataInputStream(fis);
Tuple after = tf.newTuple();
after.readFields(in);
Object o = after.get(0);
assertTrue("isa InternalMap", o instanceof InternalMap);
InternalMap m = (InternalMap)o;
assertEquals("world", (String)m.get(new Integer(1)));
assertEquals("all", (String)m.get(new Long(3L)));
assertNull(m.get("fred"));
file.delete();
}
@Test
public void testTupleToString() throws Exception {
Tuple t = giveMeOneOfEach();
assertEquals(
"toString",
"((3,3.0),{(4),(mary had a little lamb)},[hello#world,goodbye#all],42,5000000000,3.1415927,2.99792458E8,true,hello,goodbye,)",
t.toString());
}
@Test
public void testTupleHashCode() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = tf.newTuple(2);
t1.set(0, new DataByteArray("hello world"));
t1.set(1, new Integer(1));
Tuple t2 = tf.newTuple(2);
t2.set(0, new DataByteArray("hello world"));
t2.set(1, new Integer(1));
assertEquals("same data", t1.hashCode(), t2.hashCode());
Tuple t3 = tf.newTuple(3);
t3.set(0, new DataByteArray("hello world"));
t3.set(1, new Integer(1));
t3.set(2, new Long(4));
assertFalse("different size", t1.hashCode() == t3.hashCode());
Tuple t4 = tf.newTuple(2);
t4.set(0, new DataByteArray("hello world"));
t4.set(1, new Integer(2));
assertFalse("same size, different data", t1.hashCode() == t4.hashCode());
// Make sure we can take the hash code of all the types.
Tuple t5 = giveMeOneOfEach();
t5.hashCode();
}
@Test
public void testTupleEquals() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = tf.newTuple();
Tuple t2 = tf.newTuple();
t1.append(new Integer(3));
t2.append(new Integer(3));
assertFalse("different object", t1.equals(new String()));
assertTrue("same data", t1.equals(t2));
t2 = tf.newTuple();
t2.append(new Integer(4));
assertFalse("different data", t1.equals(t2));
t2 = tf.newTuple();
t2.append(new Integer(3));
t2.append(new Integer(3));
assertFalse("different size", t1.equals(t2));
}
@Test
public void testTupleCompareTo() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = tf.newTuple();
Tuple t2 = tf.newTuple();
t1.append(new Integer(3));
t2.append(new Integer(3));
assertEquals("same data equal", 0, t1.compareTo(t2));
t2 = tf.newTuple();
t2.append(new Integer(2));
assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));
t2 = tf.newTuple();
t2.append(new Integer(4));
assertTrue("less than tuple with greater value", 0 > t1.compareTo(t2));
t2 = tf.newTuple();
t2.append(new Integer(3));
t2.append(new Integer(4));
assertTrue("less than bigger tuple", 0 > t1.compareTo(t2));
t2 = tf.newTuple();
assertTrue("greater than smaller tuple", 0 < t1.compareTo(t2));
}
@Test
public void testMultiFieldTupleCompareTo() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = tf.newTuple();
Tuple t2 = tf.newTuple();
t1.append(new DataByteArray("bbb"));
t1.append(new DataByteArray("bbb"));
t2.append(new DataByteArray("bbb"));
t2.append(new DataByteArray("bbb"));
assertEquals("same data equal", 0, t1.compareTo(t2));
t2 = tf.newTuple();
t2.append(new DataByteArray("aaa"));
t2.append(new DataByteArray("aaa"));
assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));
t2 = tf.newTuple();
t2.append(new DataByteArray("ddd"));
t2.append(new DataByteArray("ddd"));
assertTrue("less than tuple with greater value", 0 > t1.compareTo(t2));
// First column same, second lesser
t2 = tf.newTuple();
t2.append(new DataByteArray("bbb"));
t2.append(new DataByteArray("aaa"));
assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));
// First column same, second greater
t2 = tf.newTuple();
t2.append(new DataByteArray("bbb"));
t2.append(new DataByteArray("ccc"));
assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));
// First column less, second same
t2 = tf.newTuple();
t2.append(new DataByteArray("aaa"));
t2.append(new DataByteArray("bbb"));
assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));
// First column greater, second same
t2 = tf.newTuple();
t2.append(new DataByteArray("ccc"));
t2.append(new DataByteArray("bbb"));
assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));
// First column less, second greater
t2 = tf.newTuple();
t2.append(new DataByteArray("aaa"));
t2.append(new DataByteArray("ccc"));
assertTrue("greater than tuple with lesser value", 0 < t1.compareTo(t2));
// First column greater, second same
t2 = tf.newTuple();
t2.append(new DataByteArray("ccc"));
t2.append(new DataByteArray("aaa"));
assertTrue("greater than tuple with lesser value", 0 > t1.compareTo(t2));
}
@Test
public void testByteArrayToString() throws Exception {
DataByteArray ba = new DataByteArray("hello world");
assertEquals("toString", "hello world", ba.toString());
}
@Test
public void testByteArrayHashCode() throws Exception {
DataByteArray ba1 = new DataByteArray("hello world");
DataByteArray ba2 = new DataByteArray("hello world");
DataByteArray ba3 = new DataByteArray("goodbye world");
assertEquals("same data", ba1.hashCode(), ba2.hashCode());
assertFalse("different data", ba1.hashCode() == ba3.hashCode());
}
@Test
public void testByteArrayEquals() throws Exception {
DataByteArray ba1 = new DataByteArray("hello world");
DataByteArray ba2 = new DataByteArray("hello world");
DataByteArray ba3 = new DataByteArray("goodbye world");
assertTrue("same data", ba1.equals(ba2));
assertFalse("different data", ba1.equals(ba3));
}
@Test
public void testByteArrayCompareTo() throws Exception {
DataByteArray ba1 = new DataByteArray("hello world");
DataByteArray ba2 = new DataByteArray("hello world");
DataByteArray ba3 = new DataByteArray("goodbye world");
assertTrue("same data", ba1.compareTo(ba2) == 0);
assertTrue("different length lexically lower value less than",
ba3.compareTo(ba1) < 0);
assertTrue("different length lexically higher value greater than",
ba1.compareTo(ba3) > 0);
ba2 = new DataByteArray("hello worlc");
assertTrue("same length lexically lower value less than",
ba2.compareTo(ba1) < 0);
assertTrue("same length lexically higher value greater than",
ba1.compareTo(ba2) > 0);
ba2 = new DataByteArray("hello worlds");
assertTrue("shorter lexically same value less than",
ba1.compareTo(ba2) < 0);
assertTrue("longer lexically same value greater than",
ba2.compareTo(ba1) > 0);
}
@Test(expected = ExecException.class)
public void testIntegerConversionErr() throws Exception {
List list = new ArrayList();
try {
DataType.toInteger(list);
} catch (ExecException ee) {
assertEquals(1071, ee.getErrorCode());
throw ee;
}
}
@Test(expected = ExecException.class)
public void testIntegerConversionErr1() throws Exception {
DataByteArray ba = new DataByteArray("hello world");
try {
DataType.toInteger(ba);
} catch (ExecException ee) {
assertEquals(1074, ee.getErrorCode());
throw ee;
}
}
@Test(expected = ExecException.class)
public void testTupleConversionErr() throws Exception {
List list = new ArrayList();
try {
DataType.toTuple(list);
} catch (ExecException ee) {
assertEquals(1071, ee.getErrorCode());
throw ee;
}
}
@Test(expected = ExecException.class)
public void testTupleConversionErr1() throws Exception {
DataByteArray ba = new DataByteArray("hello world");
try {
DataType.toTuple(ba);
} catch (ExecException ee) {
assertEquals(1071, ee.getErrorCode());
throw ee;
}
}
@Test
public void testByteArrayAppend() throws Exception {
DataByteArray expected = new DataByteArray("hello world");
DataByteArray db1 = new DataByteArray("hello ");
DataByteArray db2 = new DataByteArray("world");
db1.append(db2);
assertEquals("appends as expected", db1, expected);
}
@Test
public void testByteArrayAppendMore() throws Exception {
DataByteArray expected = new DataByteArray("hello world!");
DataByteArray db1 = new DataByteArray("hello ");
DataByteArray db2 = new DataByteArray("world");
DataByteArray db3 = new DataByteArray("!");
db1.append(db2).append(db3);
assertEquals("appends as expected", db1, expected);
}
@Test
public void testByteArrayAppendBytes() throws Exception {
DataByteArray expected = new DataByteArray("hello world");
DataByteArray db1 = new DataByteArray("hello ");
byte[] db2 = "world".getBytes();
db1.append(db2);
assertEquals("appends as expected", db1, expected);
}
@Test
public void testByteArrayAppendString() throws Exception {
DataByteArray expected = new DataByteArray("hello world");
DataByteArray db1 = new DataByteArray("hello ");
db1.append("world");
assertEquals("appends as expected", db1, expected);
}
@Test(expected = ExecException.class)
public void testMapConversionErr() throws Exception {
List list = new ArrayList();
try {
DataType.toMap(list);
} catch (ExecException ee) {
assertEquals(1071, ee.getErrorCode());
throw ee;
}
}
@Test
public void testMapConversion() throws Exception {
Map<Integer, Float> map = new HashMap<Integer, Float>();
DataType.toMap(map);
}
@Test(expected = ExecException.class)
public void testDetermineFieldSchemaErr() throws Exception {
List list = new ArrayList();
try {
DataType.determineFieldSchema(list);
fail("Error expected.");
} catch (ExecException ee) {
assertEquals(1073, ee.getErrorCode());
throw ee;
}
}
private Tuple giveMeOneOfEach() throws Exception {
TupleFactory tf = TupleFactory.getInstance();
Tuple t1 = tf.newTuple(11);
Tuple t2 = tf.newTuple(2);
t2.set(0, new Integer(3));
t2.set(1, new Float(3.0));
DataBag bag = BagFactory.getInstance().newDefaultBag();
bag.add(tf.newTuple(new Integer(4)));
bag.add(tf.newTuple(new String("mary had a little lamb")));
Map<String, Object> map = new LinkedHashMap<String, Object>(2);
map.put(new String("hello"), new String("world"));
map.put(new String("goodbye"), new String("all"));
t1.set(0, t2);
t1.set(1, bag);
t1.set(2, map);
t1.set(3, new Integer(42));
t1.set(4, new Long(5000000000L));
t1.set(5, new Float(3.141592654));
t1.set(6, new Double(2.99792458e8));
t1.set(7, new Boolean(true));
t1.set(8, new DataByteArray("hello"));
t1.set(9, new String("goodbye"));
return t1;
}
}