package com.esri.hadoop.hive.serde;
import org.junit.Assert;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.LazyString;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import com.esri.core.geometry.Geometry;
import com.esri.core.geometry.Point;
import com.esri.core.geometry.SpatialReference;
import com.esri.core.geometry.ogc.OGCGeometry;
import com.esri.hadoop.shims.HiveShims;
// Ideally tests to cover:
// - attributes and/or geometry
// - null attributes and values to not linger
// - null geometry
// - spatial reference preserved
public class TestEsriJsonSerDe extends JsonSerDeTestingBase {
@Test
public void TestIntWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{"num":7}}
addWritable(stuff, 7);
Writable jsw = jserde.serialize(stuff, rowOI);
JsonNode jn = new ObjectMapper().readTree(((Text)jsw).toString());
jn = jn.findValue("attributes");
jn = jn.findValue("num");
Assert.assertEquals(7, jn.getIntValue());
}
@Test
public void TestPointWrite() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{},"geometry":{"x":15.0,"y":5.0}}
addWritable(stuff, new Point(15.0, 5.0));
Writable jsw = jserde.serialize(stuff, rowOI);
String rslt = ((Text)jsw).toString();
JsonNode jn = new ObjectMapper().readTree(rslt);
jn = jn.findValue("geometry");
Assert.assertNotNull(jn.findValue("x"));
Assert.assertNotNull(jn.findValue("y"));
}
@Test
public void TestIntParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
SerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":null}");
value.set("{\"attributes\":{\"num\":7}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("num");
Object fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
value.set("{\"attributes\":{\"num\":9}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("num");
fieldData = rowOI.getStructFieldData(row, f0);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
@Test
public void TestPointParse() throws Exception {
Configuration config = new Configuration();
Text value = new Text();
SerDe jserde = new EsriJsonSerDe();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
Object row = jserde.deserialize(value);
StructField f0 = rowOI.getStructFieldRef("shape");
Object fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
row = jserde.deserialize(value);
f0 = rowOI.getStructFieldRef("shape");
fieldData = rowOI.getStructFieldData(row, f0);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
@Test
public void TestIntOnly() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
addWritable(stuff, 7);
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
// value.set("{\"attributes\":{\"num\":9}}");
stuff.clear();
addWritable(stuff, 9);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(9, ((IntWritable)fieldData).get());
}
@Test
public void TestPointOnly() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("shape", row, rowOI);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
//value.set("{\"attributes\":{},\"geometry\":{\"x\":7.0,\"y\":4.0}}");
stuff.clear();
addWritable(stuff, new Point(7.0, 4.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 4.0), (BytesWritable)fieldData);
}
@Test
public void TestIntPoint() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, 7L);
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((LongWritable)fieldData).get());
//value.set("{\"attributes\":{\"num\":4},\"geometry\":{\"x\":7.0,\"y\":2.0}}");
stuff.clear();
addWritable(stuff, 4L);
addWritable(stuff, new Point(7.0, 2.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(4, ((LongWritable)fieldData).get());
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
@Test
public void TestNullAttr() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "int");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7}}");
addWritable(stuff, 7);
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((IntWritable)fieldData).get());
//value.set("{\"attributes\":{}}");
stuff.set(0, null);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertNull(fieldData);
}
@Test
public void TestNullGeom() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "binary");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("shape", row, rowOI);
ckPoint(new Point(15.0, 5.0), (BytesWritable)fieldData);
//value.set("{\"attributes\":{},\"geometry\":null}");
stuff.set(0, null);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("shape", row, rowOI);
Assert.assertNull(fieldData);
}
@Test
public void TestColumnTypes() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "flag,num1,num2,text");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "boolean,tinyint,smallint,string");
SerDe jserde = mkSerDe(proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
// {"attributes":{"flag":false,"num":"5","text":"Point(15.0 5.0)"}}
addWritable(stuff, false);
addWritable(stuff, (byte)2);
addWritable(stuff, (short)5);
addWritable(stuff, "Point(15.0 5.0)");
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("flag", row, rowOI);
Assert.assertEquals(false, ((BooleanWritable)fieldData).get());
fieldData = getField("num1", row, rowOI);
Assert.assertEquals((byte)2, ((ByteWritable)fieldData).get());
fieldData = getField("num2", row, rowOI);
Assert.assertEquals((short)5, ((ShortWritable)fieldData).get());
fieldData = getField("text", row, rowOI);
Assert.assertEquals("Point(15.0 5.0)", ((Text)fieldData).toString());
stuff.set(0, new BooleanWritable(true));
stuff.set(1, new ByteWritable((byte)4));
stuff.set(2, new ShortWritable((short)4));
//stuff.set(3, new Text("other"));
LazyStringObjectInspector loi = LazyPrimitiveObjectInspectorFactory.
getLazyStringObjectInspector(false, (byte)'\0');
LazyString lstr = new LazyString(loi);
ByteArrayRef bar = new ByteArrayRef();
bar.setData("other".getBytes());
lstr.init(bar, 0, 5);
stuff.set(3, lstr);
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("flag", row, rowOI);
Assert.assertEquals(true, ((BooleanWritable)fieldData).get());
fieldData = getField("num1", row, rowOI);
Assert.assertEquals((byte)4, ((ByteWritable)fieldData).get());
fieldData = getField("num2", row, rowOI);
Assert.assertEquals((short)4, ((ShortWritable)fieldData).get());
fieldData = getField("text", row, rowOI);
Assert.assertEquals("other", ((Text)fieldData).toString());
}
@Deprecated
@Test
public void LegacyName() throws Exception {
ArrayList<Object> stuff = new ArrayList<Object>();
Properties proptab = new Properties();
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMNS, "num,shape");
proptab.setProperty(HiveShims.serdeConstants.LIST_COLUMN_TYPES, "bigint,binary");
Configuration config = new Configuration();
SerDe jserde = new JsonSerde();
jserde.initialize(config, proptab);
StructObjectInspector rowOI = (StructObjectInspector)jserde.getObjectInspector();
//value.set("{\"attributes\":{\"num\":7},\"geometry\":{\"x\":15.0,\"y\":5.0}}");
addWritable(stuff, 7L);
addWritable(stuff, new Point(15.0, 5.0));
Object row = runSerDe(stuff, jserde, rowOI);
Object fieldData = getField("num", row, rowOI);
Assert.assertEquals(7, ((LongWritable)fieldData).get());
//value.set("{\"attributes\":{\"num\":4},\"geometry\":{\"x\":7.0,\"y\":2.0}}");
stuff.clear();
addWritable(stuff, 4L);
addWritable(stuff, new Point(7.0, 2.0));
row = runSerDe(stuff, jserde, rowOI);
fieldData = getField("num", row, rowOI);
Assert.assertEquals(4, ((LongWritable)fieldData).get());
fieldData = getField("shape", row, rowOI);
ckPoint(new Point(7.0, 2.0), (BytesWritable)fieldData);
}
private SerDe mkSerDe(Properties proptab) throws Exception {
Configuration config = new Configuration();
SerDe jserde = new EsriJsonSerDe();
jserde.initialize(config, proptab);
return jserde;
}
}