/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.serde2.thrift;
import java.io.EOFException;
import java.nio.charset.CharacterCodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.io.Text;
import org.apache.thrift.TException;
import org.apache.thrift.protocol.TField;
import org.apache.thrift.protocol.TList;
import org.apache.thrift.protocol.TMap;
import org.apache.thrift.protocol.TMessage;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.protocol.TProtocolFactory;
import org.apache.thrift.protocol.TSet;
import org.apache.thrift.protocol.TStruct;
import org.apache.thrift.protocol.TType;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportException;
/**
*
* An implementation of the Thrift Protocol for ctl separated records. This is
* not thrift compliant in that it doesn't write out field ids so things cannot
* actually be versioned.
*/
public class TCTLSeparatedProtocol extends TProtocol implements
ConfigurableTProtocol, WriteNullsProtocol, SkippableTProtocol {
static final Logger LOG = LoggerFactory.getLogger(TCTLSeparatedProtocol.class
.getName());
static byte ORDERED_TYPE = (byte) -1;
/**
* Factory for JSON protocol objects.
*/
public static class Factory implements TProtocolFactory {
public TProtocol getProtocol(TTransport trans) {
return new TCTLSeparatedProtocol(trans);
}
}
/**
* These are defaults, but for now leaving them like this.
*/
protected static final String defaultPrimarySeparator = "\001";
protected static final String defaultSecondarySeparator = "\002";
protected static final String defaultRowSeparator = "\n";
protected static final String defaultMapSeparator = "\003";
/**
* The separators for this instance.
*/
protected String primarySeparator;
protected String secondarySeparator;
protected String rowSeparator;
protected String mapSeparator;
protected Pattern primaryPattern;
protected Pattern secondaryPattern;
protected Pattern mapPattern;
/**
* The quote character when supporting quotes with ability to not split across
* quoted entries. Like csv. Note that escaping the quote is not currently
* supported.
*/
protected String quote;
/**
* Inspect the separators this instance is configured with.
*/
public String getPrimarySeparator() {
return primarySeparator;
}
public String getSecondarySeparator() {
return secondarySeparator;
}
public String getRowSeparator() {
return rowSeparator;
}
public String getMapSeparator() {
return mapSeparator;
}
/**
* The transport stream is tokenized on the row separator.
*/
protected SimpleTransportTokenizer transportTokenizer;
/**
* For a single row, the split on the primary separator.
*/
protected String[] columns;
/**
* An index into what column we're on.
*/
protected int index;
/**
* For a single column, a split on the secondary separator.
*/
protected String[] fields;
/**
* An index into what field within a column we're on.
*/
protected int innerIndex;
/**
* Is this the first field we're writing.
*/
protected boolean firstField;
/**
* Is this the first list/map/set field we're writing for the current element.
*/
protected boolean firstInnerField;
/**
* Are we writing a map and need to worry about k/v separator?
*/
protected boolean isMap;
/**
* For writes, on what element are we on so we know when to use normal list
* separator or for a map know when to use the k/v separator.
*/
protected long elemIndex;
/**
* Are we currently on the top-level columns or parsing a column itself.
*/
protected boolean inner;
/**
* For places where the separators are back to back, should we return a null
* or an empty string since it is ambiguous. This also applies to extra
* columns that are read but aren't in the current record.
*/
protected boolean returnNulls;
/**
* The transport being wrapped.
*
*/
protected final TTransport innerTransport;
/**
* Strings used to lookup the various configurable paramaters of this
* protocol.
*/
public static final String ReturnNullsKey = "separators.return_nulls";
public static final String BufferSizeKey = "separators.buffer_size";
/**
* The size of the internal buffer to use.
*/
protected int bufferSize;
/**
* The string representing nulls in the serialized data. e.g., \N as in mysql.
*/
protected String nullString;
/**
* The nullString in UTF-8 bytes.
*/
protected Text nullText;
/**
* A convenience class for tokenizing a TTransport.
*/
class SimpleTransportTokenizer {
TTransport trans;
StringTokenizer tokenizer;
final String separator;
byte[] buf;
public SimpleTransportTokenizer(TTransport trans, String separator,
int buffer_length) {
this.trans = trans;
this.separator = separator;
buf = new byte[buffer_length];
}
private void initialize() {
// do not fill tokenizer until user requests since filling it could read
// in data
// not meant for this instantiation.
try {
fillTokenizer();
} catch (Exception e) {
LOG.warn("Unable to initialize tokenizer", e);
}
}
private boolean fillTokenizer() {
try {
int length = trans.read(buf, 0, buf.length);
if (length <= 0) {
tokenizer = new StringTokenizer("", separator, true);
return false;
}
String row;
try {
row = Text.decode(buf, 0, length);
} catch (CharacterCodingException e) {
throw new RuntimeException(e);
}
tokenizer = new StringTokenizer(row, separator, true);
} catch (TTransportException e) {
if(e.getType() == TTransportException.END_OF_FILE){
tokenizer = new StringTokenizer("", separator, true);
return false;
}
tokenizer = null;
throw new RuntimeException(e);
}
return true;
}
public String nextToken() throws EOFException {
StringBuilder ret = null;
boolean done = false;
if (tokenizer == null) {
fillTokenizer();
}
while (!done) {
if (!tokenizer.hasMoreTokens()) {
if (!fillTokenizer()) {
break;
}
}
try {
final String nextToken = tokenizer.nextToken();
if (nextToken.equals(separator)) {
done = true;
} else if (ret == null) {
ret = new StringBuilder(nextToken);
} else {
ret.append(nextToken);
}
} catch (NoSuchElementException e) {
if (ret == null) {
throw new EOFException(e.getMessage());
}
done = true;
}
} // while ! done
final String theRet = ret == null ? null : ret.toString();
return theRet;
}
};
/**
* The simple constructor which assumes ctl-a, ctl-b and '\n' separators and
* to return empty strings for empty fields.
*
* @param trans
* - the ttransport to use as input or output
*
*/
public TCTLSeparatedProtocol(TTransport trans) {
this(trans, defaultPrimarySeparator, defaultSecondarySeparator,
defaultMapSeparator, defaultRowSeparator, true, 4096);
}
public TCTLSeparatedProtocol(TTransport trans, int buffer_size) {
this(trans, defaultPrimarySeparator, defaultSecondarySeparator,
defaultMapSeparator, defaultRowSeparator, true, buffer_size);
}
/**
* @param trans
* - the ttransport to use as input or output
* @param primarySeparator
* the separator between columns (aka fields)
* @param secondarySeparator
* the separator within a field for things like sets and maps and
* lists
* @param mapSeparator
* - the key/value separator
* @param rowSeparator
* - the record separator
* @param returnNulls
* - whether to return a null or an empty string for fields that seem
* empty (ie two primary separators back to back)
*/
public TCTLSeparatedProtocol(TTransport trans, String primarySeparator,
String secondarySeparator, String mapSeparator, String rowSeparator,
boolean returnNulls, int bufferSize) {
super(trans);
this.returnNulls = returnNulls;
this.primarySeparator = primarySeparator;
this.secondarySeparator = secondarySeparator;
this.rowSeparator = rowSeparator;
this.mapSeparator = mapSeparator;
innerTransport = trans;
this.bufferSize = bufferSize;
nullString = "\\N";
}
/**
* Sets the internal separator patterns and creates the internal tokenizer.
*/
protected void internalInitialize() {
// in the future could allow users to specify a quote character that doesn't
// need escaping but for now ...
final String primaryPatternString = quote == null ? primarySeparator
: "(?:^|" + primarySeparator + ")(" + quote + "(?:[^" + quote + "]+|"
+ quote + quote + ")*" + quote + "|[^" + primarySeparator + "]*)";
if (quote != null) {
stripSeparatorPrefix = Pattern.compile("^" + primarySeparator);
stripQuotePrefix = Pattern.compile("^" + quote);
stripQuotePostfix = Pattern.compile(quote + "$");
}
primaryPattern = Pattern.compile(primaryPatternString);
secondaryPattern = Pattern.compile(secondarySeparator);
mapPattern = Pattern.compile(secondarySeparator + "|" + mapSeparator);
nullText = new Text(nullString);
transportTokenizer = new SimpleTransportTokenizer(innerTransport,
rowSeparator, bufferSize);
transportTokenizer.initialize();
}
/**
* For quoted fields, strip away the quotes and also need something to strip
* away the control separator when using complex split method defined here.
*/
protected Pattern stripSeparatorPrefix;
protected Pattern stripQuotePrefix;
protected Pattern stripQuotePostfix;
/**
*
* Split the line based on a complex regex pattern.
*
* @param line
* the current row
* @param p
* the pattern for matching fields in the row
* @return List of Strings - not including the separator in them
*/
protected String[] complexSplit(String line, Pattern p) {
ArrayList<String> list = new ArrayList<String>();
Matcher m = p.matcher(line);
// For each field
while (m.find()) {
String match = m.group();
if (match == null) {
break;
}
if (match.length() == 0) {
match = null;
} else {
if (stripSeparatorPrefix.matcher(match).find()) {
match = match.substring(1);
}
if (stripQuotePrefix.matcher(match).find()) {
match = match.substring(1);
}
if (stripQuotePostfix.matcher(match).find()) {
match = match.substring(0, match.length() - 1);
}
}
list.add(match);
}
return list.toArray(new String[1]);
}
protected String getByteValue(String altValue, String defaultVal) {
if (altValue != null && altValue.length() > 0) {
try {
byte[] b = new byte[1];
b[0] = Byte.parseByte(altValue);
return new String(b);
} catch (NumberFormatException e) {
return altValue;
}
}
return defaultVal;
}
/**
* Initialize the TProtocol.
*
* @param conf
* System properties
* @param tbl
* table properties
* @throws TException
*/
public void initialize(Configuration conf, Properties tbl) throws TException {
primarySeparator = getByteValue(tbl.getProperty(serdeConstants.FIELD_DELIM),
primarySeparator);
secondarySeparator = getByteValue(tbl
.getProperty(serdeConstants.COLLECTION_DELIM), secondarySeparator);
rowSeparator = getByteValue(tbl.getProperty(serdeConstants.LINE_DELIM),
rowSeparator);
mapSeparator = getByteValue(tbl.getProperty(serdeConstants.MAPKEY_DELIM),
mapSeparator);
returnNulls = Boolean.parseBoolean(
tbl.getProperty(ReturnNullsKey, String.valueOf(returnNulls)));
bufferSize = Integer.parseInt(
tbl.getProperty(BufferSizeKey, String.valueOf(bufferSize)));
nullString = tbl.getProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N");
quote = tbl.getProperty(serdeConstants.QUOTE_CHAR, null);
internalInitialize();
}
@Override
public void writeMessageBegin(TMessage message) throws TException {
}
@Override
public void writeMessageEnd() throws TException {
}
@Override
public void writeStructBegin(TStruct struct) throws TException {
firstField = true;
}
@Override
public void writeStructEnd() throws TException {
// We don't write rowSeparatorByte because that should be handled by file
// format.
}
@Override
public void writeFieldBegin(TField field) throws TException {
if (!firstField) {
internalWriteString(primarySeparator);
}
firstField = false;
}
@Override
public void writeFieldEnd() throws TException {
}
@Override
public void writeFieldStop() {
}
@Override
public void writeMapBegin(TMap map) throws TException {
// nesting not allowed!
if (map.keyType == TType.STRUCT || map.keyType == TType.MAP
|| map.keyType == TType.LIST || map.keyType == TType.SET) {
throw new TException("Not implemented: nested structures");
}
// nesting not allowed!
if (map.valueType == TType.STRUCT || map.valueType == TType.MAP
|| map.valueType == TType.LIST || map.valueType == TType.SET) {
throw new TException("Not implemented: nested structures");
}
firstInnerField = true;
isMap = true;
inner = true;
elemIndex = 0;
}
@Override
public void writeMapEnd() throws TException {
isMap = false;
inner = false;
}
@Override
public void writeListBegin(TList list) throws TException {
if (list.elemType == TType.STRUCT || list.elemType == TType.MAP
|| list.elemType == TType.LIST || list.elemType == TType.SET) {
throw new TException("Not implemented: nested structures");
}
firstInnerField = true;
inner = true;
}
@Override
public void writeListEnd() throws TException {
inner = false;
}
@Override
public void writeSetBegin(TSet set) throws TException {
if (set.elemType == TType.STRUCT || set.elemType == TType.MAP
|| set.elemType == TType.LIST || set.elemType == TType.SET) {
throw new TException("Not implemented: nested structures");
}
firstInnerField = true;
inner = true;
}
@Override
public void writeSetEnd() throws TException {
inner = false;
}
@Override
public void writeBool(boolean b) throws TException {
writeString(String.valueOf(b));
}
// for writing out single byte
private final byte[] buf = new byte[1];
@Override
public void writeByte(byte b) throws TException {
buf[0] = b;
trans_.write(buf);
}
@Override
public void writeI16(short i16) throws TException {
writeString(String.valueOf(i16));
}
@Override
public void writeI32(int i32) throws TException {
writeString(String.valueOf(i32));
}
@Override
public void writeI64(long i64) throws TException {
writeString(String.valueOf(i64));
}
@Override
public void writeDouble(double dub) throws TException {
writeString(String.valueOf(dub));
}
Text tmpText = new Text();
public void internalWriteString(String str) throws TException {
if (str != null) {
tmpText.set(str);
trans_.write(tmpText.getBytes(), 0, tmpText.getLength());
} else {
trans_.write(nullText.getBytes(), 0, nullText.getLength());
}
}
@Override
public void writeString(String str) throws TException {
if (inner) {
if (!firstInnerField) {
// super hack city notice the mod plus only happens after firstfield
// hit, so == 0 is right.
if (isMap && elemIndex++ % 2 == 0) {
internalWriteString(mapSeparator);
} else {
internalWriteString(secondarySeparator);
}
} else {
firstInnerField = false;
}
}
internalWriteString(str);
}
@Override
public void writeBinary(ByteBuffer bin) throws TException {
throw new TException(
"Ctl separated protocol cannot support writing Binary data!");
}
@Override
public TMessage readMessageBegin() throws TException {
return new TMessage();
}
@Override
public void readMessageEnd() throws TException {
}
@Override
public TStruct readStructBegin() throws TException {
assert (!inner);
try {
final String tmp = transportTokenizer.nextToken();
columns = quote == null ? primaryPattern.split(tmp) : complexSplit(tmp,
primaryPattern);
index = 0;
return new TStruct();
} catch (EOFException e) {
return null;
}
}
@Override
public void readStructEnd() throws TException {
columns = null;
}
/**
* Skip past the current field Just increments the field index counter.
*/
public void skip(byte type) {
if (inner) {
innerIndex++;
} else {
index++;
}
}
@Override
public TField readFieldBegin() throws TException {
assert (!inner);
TField f = new TField("", ORDERED_TYPE, (short) -1);
// slight hack to communicate to DynamicSerDe that the field ids are not
// being set but things are ordered.
return f;
}
@Override
public void readFieldEnd() throws TException {
fields = null;
}
@Override
public TMap readMapBegin() throws TException {
assert (!inner);
TMap map = new TMap();
if (columns[index] == null || columns[index].equals(nullString)) {
index++;
if (returnNulls) {
return null;
}
} else if (columns[index].isEmpty()) {
index++;
} else {
fields = mapPattern.split(columns[index++]);
map = new TMap(ORDERED_TYPE, ORDERED_TYPE, fields.length / 2);
}
innerIndex = 0;
inner = true;
isMap = true;
return map;
}
@Override
public void readMapEnd() throws TException {
inner = false;
isMap = false;
}
@Override
public TList readListBegin() throws TException {
assert (!inner);
TList list = new TList();
if (columns[index] == null || columns[index].equals(nullString)) {
index++;
if (returnNulls) {
return null;
}
} else if (columns[index].isEmpty()) {
index++;
} else {
fields = secondaryPattern.split(columns[index++]);
list = new TList(ORDERED_TYPE, fields.length);
}
innerIndex = 0;
inner = true;
return list;
}
@Override
public void readListEnd() throws TException {
inner = false;
}
@Override
public TSet readSetBegin() throws TException {
assert (!inner);
TSet set = new TSet();
if (columns[index] == null || columns[index].equals(nullString)) {
index++;
if (returnNulls) {
return null;
}
} else if (columns[index].isEmpty()) {
index++;
} else {
fields = secondaryPattern.split(columns[index++]);
set = new TSet(ORDERED_TYPE, fields.length);
}
inner = true;
innerIndex = 0;
return set;
}
protected boolean lastPrimitiveWasNullFlag;
public boolean lastPrimitiveWasNull() throws TException {
return lastPrimitiveWasNullFlag;
}
public void writeNull() throws TException {
writeString(null);
}
@Override
public void readSetEnd() throws TException {
inner = false;
}
@Override
public boolean readBool() throws TException {
String val = readString();
lastPrimitiveWasNullFlag = val == null;
return val == null || val.isEmpty() ? false : Boolean.parseBoolean(val);
}
@Override
public byte readByte() throws TException {
String val = readString();
lastPrimitiveWasNullFlag = val == null;
try {
return val == null || val.isEmpty() ? 0 : Byte.parseByte(val);
} catch (NumberFormatException e) {
lastPrimitiveWasNullFlag = true;
return 0;
}
}
@Override
public short readI16() throws TException {
String val = readString();
lastPrimitiveWasNullFlag = val == null;
try {
return val == null || val.isEmpty() ? 0 : Short.parseShort(val);
} catch (NumberFormatException e) {
lastPrimitiveWasNullFlag = true;
return 0;
}
}
@Override
public int readI32() throws TException {
String val = readString();
lastPrimitiveWasNullFlag = val == null;
try {
return val == null || val.isEmpty() ? 0 : Integer.parseInt(val);
} catch (NumberFormatException e) {
lastPrimitiveWasNullFlag = true;
return 0;
}
}
@Override
public long readI64() throws TException {
String val = readString();
lastPrimitiveWasNullFlag = val == null;
try {
return val == null || val.isEmpty() ? 0 : Long.parseLong(val);
} catch (NumberFormatException e) {
lastPrimitiveWasNullFlag = true;
return 0;
}
}
@Override
public double readDouble() throws TException {
String val = readString();
lastPrimitiveWasNullFlag = val == null;
try {
return val == null || val.isEmpty() ? 0 : Double.parseDouble(val);
} catch (NumberFormatException e) {
lastPrimitiveWasNullFlag = true;
return 0;
}
}
@Override
public String readString() throws TException {
String ret;
if (!inner) {
ret = columns != null && index < columns.length ? columns[index] : null;
index++;
} else {
ret = fields != null && innerIndex < fields.length ? fields[innerIndex]
: null;
innerIndex++;
}
if (ret == null || ret.equals(nullString)) {
return returnNulls ? null : "";
} else {
return ret;
}
}
@Override
public ByteBuffer readBinary() throws TException {
throw new TException("Not implemented for control separated data");
}
}