CSVInstanceWriterTest.java

/*
 * Copyright (c) 2014 Data Harmonisation Panel
 * 
 * All rights reserved. This program and the accompanying materials are made
 * available under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the License,
 * or (at your option) any later version.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this distribution. If not, see <http://www.gnu.org/licenses/>.
 * 
 * Contributors:
 *     Data Harmonisation Panel <http://www.dhpanel.eu>
 */

package eu.esdihumboldt.hale.io.csv.writer;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.File;
import java.io.FileReader;
import java.util.Iterator;
import java.util.List;

import org.eclipse.core.runtime.content.IContentType;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

import au.com.bytecode.opencsv.CSVReader;
import eu.esdihumboldt.cst.test.TransformationExample;
import eu.esdihumboldt.cst.test.TransformationExamples;
import eu.esdihumboldt.hale.common.core.HalePlatform;
import eu.esdihumboldt.hale.common.core.io.Value;
import eu.esdihumboldt.hale.common.core.io.report.IOReport;
import eu.esdihumboldt.hale.common.core.io.supplier.FileIOSupplier;
import eu.esdihumboldt.hale.common.instance.io.InstanceWriter;
import eu.esdihumboldt.hale.common.instance.model.InstanceCollection;
import eu.esdihumboldt.hale.common.test.TestUtil;
import eu.esdihumboldt.hale.io.csv.InstanceTableIOConstants;
import eu.esdihumboldt.hale.io.csv.reader.internal.CSVSchemaReader;
import eu.esdihumboldt.hale.io.csv.writer.internal.CSVInstanceWriter;

/**
 * Test class for {@link CSVInstanceWriter}
 * 
 * @author Yasmina Kammeyer
 */
public class CSVInstanceWriterTest {

	/**
	 * a temporary folder to safely store tmp files. Will be deleted after test
	 * (successfully or not)
	 */
	@Rule
	public TemporaryFolder tmpFolder = new TemporaryFolder();

	/**
	 * Wait for needed services to be running
	 */
	@BeforeClass
	public static void waitForServices() {
		TestUtil.startConversionService();
	}

	/**
	 * Test - write simple data, without nested properties
	 * 
	 * @throws Exception , if an error occurs
	 */
	@Test
	public void testWriteSimpleSchema() throws Exception {

		TransformationExample example = TransformationExamples
				.getExample(TransformationExamples.SIMPLE_ASSIGN);
		// alternative the data could be generated by iterating through the
		// exempleproject's source data
		String propertyNames = "id,a1,b1,c1";
		String firstDataRow = "id0,a10,b10,c10";
		// header size
		int numberOfEntries = 4;
		int numberOfRows = 3;
		char sep = ',';

		File tmpFile = tmpFolder.newFile("csvTestWriteSimpleSchema.csv");

		assertTrue(
				"Csv Export was not successful.",
				writeCsvToFile(tmpFile, true, Value.of(sep), null, null,
						example.getSourceInstances()));

		CSVReader reader = new CSVReader(new FileReader(tmpFile), sep);
		List<String[]> rows = reader.readAll();
		//
		reader.close();

		assertEquals("Not enough rows.", numberOfRows, rows.size());

		// Check header ###
		Iterator<String[]> row = rows.iterator();

		String[] header = row.next();
		assertEquals("There are not enough entries.", numberOfEntries, header.length);

		for (int i = 0; i < header.length; i++) {
			assertTrue("The header of the csv file do not contain all properties.",
					propertyNames.contains(header[i]));
		}

		String[] dataRow = row.next();
		for (int i = 0; i < dataRow.length; i++) {
			assertTrue("The first data row of the csv file do not contain all properties.",
					firstDataRow.contains(dataRow[i]));
		}
	}

	/**
	 * Test - write simple data, without nested properties
	 * 
	 * @throws Exception , if an error occurs
	 */
	@Test
	public void testWriteSimpleSchemaDelimiter() throws Exception {

		TransformationExample example = TransformationExamples
				.getExample(TransformationExamples.SIMPLE_ASSIGN);
		// alternative the data could be generated by iterating through the
		// exempleproject's source data
		String propertyNames = "id,a1,b1,c1";
		String firstDataRow = "id0,a10,b10,c10";
		// header size
		int numberOfEntries = 4;
		int numberOfRows = 3;
		char sep = '\t';
		char quo = '\'';
		char esc = '"';

		File tmpFile = tmpFolder.newFile("csvTestWriteSimpleSchemaDelimiter.csv");

		assertTrue(
				"Csv Export was not successful.",
				writeCsvToFile(tmpFile, true, Value.of(sep), Value.of(quo), Value.of(esc),
						example.getSourceInstances()));

		CSVReader reader = new CSVReader(new FileReader(tmpFile), sep, quo, esc);
		List<String[]> rows = reader.readAll();
		//
		reader.close();

		assertEquals("Not enough rows.", numberOfRows, rows.size());

		// Check header ###
		Iterator<String[]> row = rows.iterator();

		String[] header = row.next();
		assertEquals("There are not enough entries.", numberOfEntries, header.length);

		for (int i = 0; i < header.length; i++) {
			assertTrue("The header of the csv file do not contain all properties.",
					propertyNames.contains(header[i]));
		}

		String[] dataRow = row.next();
		for (int i = 0; i < dataRow.length; i++) {
			assertTrue(
					"The first data row of the csv file do not contain all properties. Miss on : "
							+ dataRow[i], firstDataRow.contains(dataRow[i]));
		}
	}

	/**
	 * Test - write data of complex schema and analyze result
	 * 
	 * @throws Exception , if an error occurs
	 */
	@Test
	public void testWriteComplexSchema() throws Exception {

		TransformationExample example = TransformationExamples
				.getExample(TransformationExamples.SIMPLE_COMPLEX);
		// alternative the data could be generated by iterating through the
		// exempleproject's source data
		String propertyNames = "id,name,details.age,details.income,details.address.street,details.address.city";
		String firstDataRow = "id0,name0,age0,income0,street0,city0,street1,city1";
		// String secondDataRow =
		// "id1,name1,age1,income1,street2,city2,street3,city3";
		int numberOfEntries = 6;
		int numberOfRows = 3;
		char sep = ',';

		File tmpFile = tmpFolder.newFile("csvTestWriteComplexSchema.csv");

		assertTrue(
				"Csv Export was not successful.",
				writeCsvToFile(tmpFile, true, Value.of(sep), null, null,
						example.getSourceInstances()));

		CSVReader reader = new CSVReader(new FileReader(tmpFile), sep);
		List<String[]> rows = reader.readAll();
		//
		reader.close();

		assertEquals("Not enough rows.", numberOfRows, rows.size());

		// Check header ###
		Iterator<String[]> row = rows.iterator();

		String[] header = row.next();

		assertEquals("There are not enough entries.", numberOfEntries, header.length);

		for (int i = 0; i < header.length; i++) {
			assertTrue("The header of the csv file do not contain all properties.",
					propertyNames.contains(header[i]));
			// This is for debug purposes to check which properties are missing
			// propertyNames = propertyNames.replaceFirst(header[i], "");
		}

		String[] dataRow = row.next();
		for (int i = 0; i < dataRow.length; i++) {
			assertTrue("The first data row of the csv file do not contain all properties.",
					firstDataRow.contains(dataRow[i]));
			// This is for debug purposes ...
			// firstDataRow = firstDataRow.replaceFirst(dataRow[i], "");
		}

	}

	private boolean writeCsvToFile(File tmpFile, boolean skipFirstLine, Value sep, Value quo,
			Value esc, InstanceCollection instances) throws Exception {
		// set instances to xls instance writer
		InstanceWriter writer = new CSVInstanceWriter();
		IContentType contentType = HalePlatform.getContentTypeManager().getContentType(
				"eu.esdihumboldt.hale.io.csv");
		writer.setParameter(InstanceTableIOConstants.SOLVE_NESTED_PROPERTIES,
				Value.of(skipFirstLine));
		writer.setParameter(CSVSchemaReader.PARAM_SEPARATOR, sep);
		writer.setParameter(CSVSchemaReader.PARAM_QUOTE, quo);
		writer.setParameter(CSVSchemaReader.PARAM_ESCAPE, esc);
		writer.setInstances(instances);
		// write instances to a temporary CSV file
		writer.setTarget(new FileIOSupplier(tmpFile));
		writer.setContentType(contentType);
		IOReport report = writer.execute(null);
		return report.isSuccess();
	}
}