/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.internal.csv;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Arrays;
import junit.framework.TestCase;
/**
* CSVParserTest
*
* The test are organized in three different sections:
* The 'setter/getter' section, the lexer section and finally the parser
* section. In case a test fails, you should follow a top-down approach for
* fixing a potential bug (it's likely that the parser itself fails if the lexer
* has problems...).
*/
public class CSVParserTest extends TestCase {
/**
* TestCSVParser.
*/
static class TestCSVParser extends CSVParser {
/**
* Test parser to investigate the type of the internal Token.
* @param in a Reader
*/
TestCSVParser(Reader in) {
super(in);
}
TestCSVParser(Reader in, CSVStrategy strategy) {
super(in, strategy);
}
/**
* Calls super.nextToken() and prints out a String representation of token
* type and content.
* @return String representation of token type and content
* @throws IOException like {@link CSVParser#nextToken()}
*/
public String testNextToken() throws IOException {
Token t = super.nextToken();
return Integer.toString(t.type) + ";" + t.content + ";";
}
}
// ======================================================
// lexer tests
// ======================================================
// Single line (without comment)
public void testNextToken1() throws IOException {
String code = "abc,def, hijk, lmnop, qrst,uv ,wxy ,z , ,";
TestCSVParser parser = new TestCSVParser(new StringReader(code));
assertEquals(CSVParser.TT_TOKEN + ";abc;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";def;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";hijk;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";lmnop;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";qrst;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";uv;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";wxy;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";z;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
}
// multiline including comments (and empty lines)
public void testNextToken2() throws IOException {
/* file: 1,2,3,
* a,b x,c
*
* # this is a comment
* d,e,
*
*/
String code = "1,2,3,\na,b x,c\n#foo\n\nd,e,\n\n";
CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
// strategy.setIgnoreEmptyLines(false);
strategy.setCommentStart('#');
TestCSVParser parser = new TestCSVParser(new StringReader(code), strategy);
assertEquals(CSVParser.TT_TOKEN + ";1;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";2;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";3;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";b x;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";c;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";d;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";e;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
}
// simple token with escaping
public void testNextToken3() throws IOException {
/* file: a,\,,b
* \,,
*/
String code = "a,\\,,b\n\\,,";
CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
strategy.setCommentStart('#');
TestCSVParser parser = new TestCSVParser(new StringReader(code), strategy);
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
// an unquoted single backslash is not an escape char
assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
// an unquoted single backslash is not an escape char
assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
}
// encapsulator tokenizer (sinle line)
public void testNextToken4() throws IOException {
/* file: a,"foo",b
* a, " foo",b
* a,"foo " ,b // whitespace after closing encapsulator
* a, " foo " ,b
*/
String code =
"a,\"foo\",b\na, \" foo\",b\na,\"foo \" ,b\na, \" foo \" ,b";
TestCSVParser parser = new TestCSVParser(new StringReader(code));
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";foo;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + "; foo;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";foo ;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + "; foo ;", parser.testNextToken());
// assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";b;", parser.testNextToken());
}
// encapsulator tokenizer (multi line, delimiter in string)
public void testNextToken5() throws IOException {
String code =
"a,\"foo\n\",b\n\"foo\n baar ,,,\"\n\"\n\t \n\"";
TestCSVParser parser = new TestCSVParser(new StringReader(code));
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";foo\n;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
assertEquals(CSVParser.TT_EORECORD + ";foo\n baar ,,,;",
parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";\n\t \n;", parser.testNextToken());
}
// change delimiters, comment, encapsulater
public void testNextToken6() throws IOException {
/* file: a;'b and \' more
* '
* !comment;;;;
* ;;
*/
String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
TestCSVParser parser = new TestCSVParser(new StringReader(code), new CSVStrategy(';', '\'', '!'));
assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
assertEquals(
CSVParser.TT_EORECORD + ";b and ' more\n;",
parser.testNextToken());
}
// ======================================================
// parser tests
// ======================================================
String code =
"a,b,c,d\n"
+ " a , b , 1 2 \n"
+ "\"foo baar\", b,\n"
// + " \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
+ " \"foo\n,,\n\"\",,\n\"\"\",d,e\n"; // changed to use standard CSV escaping
String[][] res = {
{"a", "b", "c", "d"},
{"a", "b", "1 2"},
{"foo baar", "b", ""},
{"foo\n,,\n\",,\n\"", "d", "e"}
};
public void testGetLine() throws IOException {
CSVParser parser = new CSVParser(new StringReader(code));
String[] tmp = null;
for (int i = 0; i < res.length; i++) {
tmp = parser.getLine();
assertTrue(Arrays.equals(res[i], tmp));
}
tmp = parser.getLine();
assertTrue(tmp == null);
}
public void testNextValue() throws IOException {
CSVParser parser = new CSVParser(new StringReader(code));
String tmp = null;
for (int i = 0; i < res.length; i++) {
for (int j = 0; j < res[i].length; j++) {
tmp = parser.nextValue();
assertEquals(res[i][j], tmp);
}
}
tmp = parser.nextValue();
assertTrue(tmp == null);
}
public void testGetAllValues() throws IOException {
CSVParser parser = new CSVParser(new StringReader(code));
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
public void testExcelStrategy1() throws IOException {
String code =
"value1,value2,value3,value4\r\na,b,c,d\r\n x,,,"
+ "\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n";
String[][] res = {
{"value1", "value2", "value3", "value4"},
{"a", "b", "c", "d"},
{" x", "", "", ""},
{""},
{"\"hello\"", " \"world\"", "abc\ndef", ""}
};
CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
public void testExcelStrategy2() throws Exception {
String code = "foo,baar\r\n\r\nhello,\r\n\r\nworld,\r\n";
String[][] res = {
{"foo", "baar"},
{""},
{"hello", ""},
{""},
{"world", ""}
};
CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
public void testEndOfFileBehaviourExcel() throws Exception {
String[] codes = {
"hello,\r\n\r\nworld,\r\n",
"hello,\r\n\r\nworld,",
"hello,\r\n\r\nworld,\"\"\r\n",
"hello,\r\n\r\nworld,\"\"",
"hello,\r\n\r\nworld,\n",
"hello,\r\n\r\nworld,",
"hello,\r\n\r\nworld,\"\"\n",
"hello,\r\n\r\nworld,\"\""
};
String[][] res = {
{"hello", ""},
{""}, // ExcelStrategy does not ignore empty lines
{"world", ""}
};
String code;
for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
code = codes[codeIndex];
CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
}
public void testEndOfFileBehaviorCSV() throws Exception {
String[] codes = {
"hello,\r\n\r\nworld,\r\n",
"hello,\r\n\r\nworld,",
"hello,\r\n\r\nworld,\"\"\r\n",
"hello,\r\n\r\nworld,\"\"",
"hello,\r\n\r\nworld,\n",
"hello,\r\n\r\nworld,",
"hello,\r\n\r\nworld,\"\"\n",
"hello,\r\n\r\nworld,\"\""
};
String[][] res = {
{"hello", ""}, // CSV Strategy ignores empty lines
{"world", ""}
};
String code;
for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
code = codes[codeIndex];
CSVParser parser = new CSVParser(new StringReader(code));
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
}
public void testEmptyLineBehaviourExcel() throws Exception {
String[] codes = {
"hello,\r\n\r\n\r\n",
"hello,\n\n\n",
"hello,\"\"\r\n\r\n\r\n",
"hello,\"\"\n\n\n"
};
String[][] res = {
{"hello", ""},
{""}, // ExcelStrategy does not ignore empty lines
{""}
};
String code;
for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
code = codes[codeIndex];
CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
}
public void testEmptyLineBehaviourCSV() throws Exception {
String[] codes = {
"hello,\r\n\r\n\r\n",
"hello,\n\n\n",
"hello,\"\"\r\n\r\n\r\n",
"hello,\"\"\n\n\n"
};
String[][] res = {
{"hello", ""} // CSV Strategy ignores empty lines
};
String code;
for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
code = codes[codeIndex];
CSVParser parser = new CSVParser(new StringReader(code));
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
}
public void OLDtestBackslashEscaping() throws IOException {
String code =
"one,two,three\n"
+ "on\\\"e,two\n"
+ "on\"e,two\n"
+ "one,\"tw\\\"o\"\n"
+ "one,\"t\\,wo\"\n"
+ "one,two,\"th,ree\"\n"
+ "\"a\\\\\"\n"
+ "a\\,b\n"
+ "\"a\\\\,b\"";
String[][] res = {
{ "one", "two", "three" },
{ "on\\\"e", "two" },
{ "on\"e", "two" },
{ "one", "tw\"o" },
{ "one", "t\\,wo" }, // backslash in quotes only escapes a delimiter (",")
{ "one", "two", "th,ree" },
{ "a\\\\" }, // backslash in quotes only escapes a delimiter (",")
{ "a\\", "b" }, // a backslash must be returnd
{ "a\\\\,b" } // backslash in quotes only escapes a delimiter (",")
};
CSVParser parser = new CSVParser(new StringReader(code));
String[][] tmp = parser.getAllValues();
assertEquals(res.length, tmp.length);
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
public void testBackslashEscaping() throws IOException {
// To avoid confusion over the need for escaping chars in java code,
// We will test with a forward slash as the escape char, and a single
// quote as the encapsulator.
String code =
"one,two,three\n" // 0
+ "'',''\n" // 1) empty encapsulators
+ "/',/'\n" // 2) single encapsulators
+ "'/'','/''\n" // 3) single encapsulators encapsulated via escape
+ "'''',''''\n" // 4) single encapsulators encapsulated via doubling
+ "/,,/,\n" // 5) separator escaped
+ "//,//\n" // 6) escape escaped
+ "'//','//'\n" // 7) escape escaped in encapsulation
+ " 8 , \"quoted \"\" /\" // string\" \n" // don't eat spaces
+ "9, /\n \n" // escaped newline
+ "";
String[][] res = {
{ "one", "two", "three" }, // 0
{ "", "" }, // 1
{ "'", "'" }, // 2
{ "'", "'" }, // 3
{ "'", "'" }, // 4
{ ",", "," }, // 5
{ "/", "/" }, // 6
{ "/", "/" }, // 7
{ " 8 ", " \"quoted \"\" \" / string\" " },
{ "9", " \n " },
};
CSVStrategy strategy = new CSVStrategy(',','\'',CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true,"\n");
CSVParser parser = new CSVParser(new StringReader(code), strategy);
String[][] tmp = parser.getAllValues();
assertTrue(tmp.length > 0);
for (int i = 0; i < res.length; i++) {
assertTrue(Arrays.equals(res[i], tmp[i]));
}
}
public void testBackslashEscaping2() throws IOException {
// To avoid confusion over the need for escaping chars in java code,
// We will test with a forward slash as the escape char, and a single
// quote as the encapsulator.
String code = ""
+ " , , \n" // 1)
+ " \t , , \n" // 2)
+ " // , /, , /,\n" // 3)
+ "";
String[][] res = {
{ " ", " ", " " }, // 1
{ " \t ", " ", " " }, // 2
{ " / ", " , ", " ," }, //3
};
CSVStrategy strategy = new CSVStrategy
(',', CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED, '/', false, false, true, true, "\n");
CSVParser parser = new CSVParser(new StringReader(code), strategy);
String[][] tmp = parser.getAllValues();
assertTrue(tmp.length > 0);
if (!CSVPrinterTest.equals(res, tmp)) {
assertTrue(false);
}
}
public void testDefaultStrategy() throws IOException {
String code = ""
+ "a,b\n" // 1)
+ "\"\n\",\" \"\n" // 2)
+ "\"\",#\n" // 2)
;
String[][] res = {
{ "a", "b" },
{ "\n", " " },
{ "", "#" },
};
CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
assertEquals(CSVStrategy.COMMENTS_DISABLED, strategy.getCommentStart());
CSVParser parser = new CSVParser(new StringReader(code), strategy);
String[][] tmp = parser.getAllValues();
assertTrue(tmp.length > 0);
if (!CSVPrinterTest.equals(res, tmp)) {
assertTrue(false);
}
String[][] res_comments = {
{ "a", "b" },
{ "\n", " " },
{ ""},
};
strategy = new CSVStrategy(',','"','#');
parser = new CSVParser(new StringReader(code), strategy);
tmp = parser.getAllValues();
if (!CSVPrinterTest.equals(res_comments, tmp)) {
assertTrue(false);
}
}
public void testUnicodeEscape() throws IOException {
String code = "abc,\\u0070\\u0075\\u0062\\u006C\\u0069\\u0063";
CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
strategy.setUnicodeEscapeInterpretation(true);
CSVParser parser = new CSVParser(new StringReader(code), strategy);
String[] data = parser.getLine();
assertEquals(2, data.length);
assertEquals("abc", data[0]);
assertEquals("public", data[1]);
}
public void testCarriageReturnLineFeedEndings() throws IOException {
String code = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
CSVParser parser = new CSVParser(new StringReader(code));
String[][] data = parser.getAllValues();
assertEquals(4, data.length);
}
public void testIgnoreEmptyLines() throws IOException {
String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
//String code = "world\r\n\n";
//String code = "foo;baar\r\n\r\nhello;\r\n\r\nworld;\r\n";
CSVParser parser = new CSVParser(new StringReader(code));
String[][] data = parser.getAllValues();
assertEquals(3, data.length);
}
public void testLineTokenConsistency() throws IOException {
String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
CSVParser parser = new CSVParser(new StringReader(code));
String[][] data = parser.getAllValues();
parser = new CSVParser(new StringReader(code));
CSVParser parser1 = new CSVParser(new StringReader(code));
for (int i = 0; i < data.length; i++) {
assertTrue(Arrays.equals(parser1.getLine(), data[i]));
for (int j = 0; j < data[i].length; j++) {
assertEquals(parser.nextValue(), data[i][j]);
}
}
}
// From SANDBOX-153
public void testDelimiterIsWhitespace() throws IOException {
String code = "one\ttwo\t\tfour \t five\t six";
TestCSVParser parser = new TestCSVParser(new StringReader(code), CSVStrategy.TDF_STRATEGY);
assertEquals(CSVParser.TT_TOKEN + ";one;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";two;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";four;", parser.testNextToken());
assertEquals(CSVParser.TT_TOKEN + ";five;", parser.testNextToken());
assertEquals(CSVParser.TT_EOF + ";six;", parser.testNextToken());
}
}