/*
* Copyright © 2014 Cask Data, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package co.cask.cdap.io;
import co.cask.cdap.api.data.schema.Schema;
import co.cask.cdap.api.data.schema.UnsupportedTypeException;
import co.cask.cdap.common.io.BinaryDecoder;
import co.cask.cdap.common.io.BinaryEncoder;
import co.cask.cdap.internal.io.DatumWriter;
import co.cask.cdap.internal.io.ReflectionDatumReader;
import co.cask.cdap.internal.io.ReflectionDatumWriter;
import co.cask.cdap.internal.io.ReflectionSchemaGenerator;
import co.cask.cdap.internal.io.TypeRepresentation;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.reflect.TypeToken;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.net.URI;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
/**
*
*/
public class DatumCodecTest {
/**
*
*/
public static class Value {
private final int id;
private final String name;
public Value(int id, String name) {
this.id = id;
this.name = name;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Value value = (Value) o;
return id == value.id && name.equals(value.name);
}
@Override
public int hashCode() {
int result = id;
result = 31 * result + name.hashCode();
return result;
}
@Override
public String toString() {
return "Value{" +
"id=" + id +
", name='" + name + '\'' +
'}';
}
}
/**
*
*/
public static class Record1 {
private final int i;
private final Map<Integer, Value> properties;
private final int[] numbers;
private final URL url;
private final UUID uuid;
private final String nullStr;
public Record1(int i, Map<Integer, Value> properties, URL url) {
this.i = i;
this.properties = properties;
this.numbers = new int[] {1, 2};
this.url = url;
this.uuid = UUID.randomUUID();
this.nullStr = null;
}
}
/**
*
*/
public static class Record2 {
private final Long i;
private final Map<String, Value> properties;
private final String name;
private final long[] numbers;
private final URI url;
private final UUID uuid;
private final String nullStr;
public Record2(long i, Map<String, Value> properties, String name) {
this.i = i;
this.properties = properties;
this.name = name;
this.numbers = new long[0];
this.url = null;
this.uuid = null;
this.nullStr = null;
}
}
@Test
public void testTypeProject() throws IOException, UnsupportedTypeException {
final Record1 r1 = new Record1(10, Maps.<Integer, Value>newHashMap(), new URL("http://www.yahoo.com"));
r1.properties.put(1, new Value(1, "Name1"));
r1.properties.put(2, new Value(2, "Name2"));
r1.properties.put(3, null);
PipedOutputStream output = new PipedOutputStream();
PipedInputStream input = new PipedInputStream(output);
Schema sourceSchema = new ReflectionSchemaGenerator().generate(Record1.class);
Schema targetSchema = new ReflectionSchemaGenerator().generate(Record2.class);
new ReflectionDatumWriter<Record1>(sourceSchema).encode(r1, new BinaryEncoder(output));
Record2 r2 = new ReflectionDatumReader<>(targetSchema, TypeToken.of(Record2.class))
.read(new BinaryDecoder(input), sourceSchema);
Assert.assertEquals(10L, r2.i.longValue());
Assert.assertTrue(Iterables.all(r2.properties.entrySet(), new Predicate<Map.Entry<String, Value>>() {
@Override
public boolean apply(Map.Entry<String, Value> input) {
Value value = r1.properties.get(Integer.valueOf(input.getKey()));
return (value == null && input.getValue() == null) || (value.equals(input.getValue()));
}
}));
Assert.assertNull(r2.name);
Assert.assertArrayEquals(new long[] {1L, 2L}, r2.numbers);
Assert.assertEquals(URI.create("http://www.yahoo.com"), r2.url);
Assert.assertEquals(r1.uuid, r2.uuid);
}
@Test
public void testCollection() throws UnsupportedTypeException, IOException {
List<String> list = Lists.newArrayList("1", "2", "3");
Schema sourceSchema = new ReflectionSchemaGenerator().generate(new TypeToken<List<String>>() { }.getType());
Schema targetSchema = new ReflectionSchemaGenerator().generate(new TypeToken<Set<String>>() { }.getType());
PipedOutputStream output = new PipedOutputStream();
PipedInputStream input = new PipedInputStream(output);
new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output));
Set<String> set = new ReflectionDatumReader<>(targetSchema, new TypeToken<Set<String>>() { })
.read(new BinaryDecoder(input), sourceSchema);
Assert.assertEquals(Sets.newHashSet("1", "2", "3"), set);
targetSchema = new ReflectionSchemaGenerator().generate(String[].class);
new ReflectionDatumWriter<List<String>>(sourceSchema).encode(list, new BinaryEncoder(output));
String[] array = new ReflectionDatumReader<>(targetSchema, new TypeToken<String[]>() { })
.read(new BinaryDecoder(input), sourceSchema);
Assert.assertArrayEquals(new String[]{"1", "2", "3"}, array);
}
/**
*
*/
public static final class Node {
int d;
Node next;
}
@Test(expected = IOException.class)
public void testCircularRef() throws UnsupportedTypeException, IOException {
Schema schema = new ReflectionSchemaGenerator().generate(Node.class);
Node head = new Node();
head.next = new Node();
head.next.next = head;
ByteArrayOutputStream output = new ByteArrayOutputStream();
new ReflectionDatumWriter<Node>(schema).encode(head, new BinaryEncoder(output));
}
/**
*
*/
public static final class MoreFields {
static final class Inner {
final Map<String, String> map;
final String b;
Inner(String b) {
this.b = b;
map = ImmutableMap.of("b", b);
}
}
final int i;
final double d;
final String k;
final List<String> list;
final Inner inner;
public MoreFields(int i, double d, String k, List<String> list) {
this.i = i;
this.d = d;
this.k = k;
this.list = list;
inner = new Inner("inner");
}
}
/**
*
*/
public static final class LessFields {
static final class Inner {
String b;
}
String k;
Inner inner;
}
@Test
public void test2() {
float f = 56L / 100F;
System.out.println(f);
}
@Test
public void testReduceProjection() throws IOException, UnsupportedTypeException {
PipedOutputStream output = new PipedOutputStream();
PipedInputStream input = new PipedInputStream(output);
Schema sourceSchema = new ReflectionSchemaGenerator().generate(MoreFields.class);
Schema targetSchema = new ReflectionSchemaGenerator().generate(LessFields.class);
MoreFields moreFields = new MoreFields(10, 20.2, "30", ImmutableList.of("1", "2"));
new ReflectionDatumWriter<MoreFields>(sourceSchema).encode(moreFields, new BinaryEncoder(output));
LessFields lessFields = new ReflectionDatumReader<>(targetSchema, TypeToken.of(LessFields.class))
.read(new BinaryDecoder(input), sourceSchema);
Assert.assertEquals("30", lessFields.k);
Assert.assertEquals(moreFields.inner.b, lessFields.inner.b);
}
/**
*
*/
public enum TestEnum {
VALUE1, VALUE2, VALUE3
}
@Test
public void testEnum() throws IOException, UnsupportedTypeException {
PipedOutputStream output = new PipedOutputStream();
PipedInputStream input = new PipedInputStream(output);
Schema schema = new ReflectionSchemaGenerator().generate(TestEnum.class);
ReflectionDatumWriter<TestEnum> writer = new ReflectionDatumWriter<>(schema);
BinaryEncoder encoder = new BinaryEncoder(output);
writer.encode(TestEnum.VALUE1, encoder);
writer.encode(TestEnum.VALUE3, encoder);
writer.encode(TestEnum.VALUE2, encoder);
BinaryDecoder decoder = new BinaryDecoder(input);
Schema readSchema = Schema.parseJson(schema.toString());
ReflectionDatumReader<TestEnum> reader = new ReflectionDatumReader<>(readSchema, TypeToken.of(TestEnum.class));
Assert.assertEquals(TestEnum.VALUE1, reader.read(decoder, readSchema));
Assert.assertEquals(TestEnum.VALUE3, reader.read(decoder, readSchema));
Assert.assertEquals(TestEnum.VALUE2, reader.read(decoder, readSchema));
}
// this tests that the datum reader treats empty fields correctly. It reproduces the issue in ENG-2404.
@Test
public void testEmptyValue() throws UnsupportedTypeException, IOException {
Schema schema = new ReflectionSchemaGenerator().generate(RecordWithString.class);
TypeRepresentation typeRep = new TypeRepresentation(RecordWithString.class);
DatumWriter<RecordWithString> datumWriter = new ReflectionDatumWriter<>(schema);
@SuppressWarnings("unchecked")
ReflectionDatumReader<RecordWithString> datumReader = new ReflectionDatumReader<>(
schema, (TypeToken<RecordWithString>) TypeToken.of(typeRep.toType()));
RecordWithString record = new RecordWithString();
record.setA(42);
record.setTheString("");
ByteArrayOutputStream bos = new ByteArrayOutputStream();
BinaryEncoder encoder = new BinaryEncoder(bos);
datumWriter.encode(record, encoder);
byte[] bytes = bos.toByteArray();
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
BinaryDecoder decoder = new BinaryDecoder(bis);
RecordWithString rec = datumReader.read(decoder, schema);
Assert.assertEquals(record.getA(), rec.getA());
Assert.assertEquals(record.getTheString(), rec.getTheString());
}
}
// dummy class for testEmptyValue()
class RecordWithString {
int a;
int getA() {
return a;
}
void setA(int a) {
this.a = a;
}
private String theString;
String getTheString() {
return theString;
}
void setTheString(String theString) {
this.theString = theString;
}
}