CSVParser.java

/*
 * Copyright 2014 Red Hat, Inc. and/or its affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.dashbuilder.dataprovider.csv;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.text.DateFormat;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;

import au.com.bytecode.opencsv.CSVReader;
import org.apache.commons.lang3.StringUtils;
import org.dashbuilder.dataset.ColumnType;
import org.dashbuilder.dataset.DataColumn;
import org.dashbuilder.dataset.DataSet;
import org.dashbuilder.dataset.DataSetFactory;
import org.dashbuilder.dataset.def.CSVDataSetDef;
import org.dashbuilder.dataset.def.DataColumnDef;

public class CSVParser {

    // Custom pattern for Unix dates (epoch).
    public static final String DATE_FORMAT_EPOCH = "epoch";
    
    protected transient Map<String,DateFormat> _dateFormatMap = new HashMap<String,DateFormat>();
    protected transient Map<String,DecimalFormat> _numberFormatMap = new HashMap<String,DecimalFormat>();
    protected CSVDataSetDef dataSetDef;
    protected CSVFileStorage csvStorage;

    public CSVParser(CSVDataSetDef def, CSVFileStorage csvFileStorage) {
        this.dataSetDef = def;
        this.csvStorage = csvFileStorage;
    }

    protected boolean isColumnIncluded(String columnId) {
        if (dataSetDef.isAllColumnsEnabled()) return true;
        if (dataSetDef.getColumns() == null) return false;
        return dataSetDef.getColumnById(columnId) != null;
    }

    protected DataSet load() throws Exception {
        InputStream is = getCSVInputStream();
        try {
            BufferedReader br = new BufferedReader(new InputStreamReader(is));
            CSVReader csvReader = new CSVReader(br, dataSetDef.getSeparatorChar(), dataSetDef.getQuoteChar(), dataSetDef.getEscapeChar());

            String[] header = csvReader.readNext();
            if (header == null) throw new IOException("CSV has no header: " + dataSetDef);

            String[] firstRow = csvReader.readNext();
            if (firstRow == null || firstRow.length < header.length) firstRow = null;

            // Build the data set structure
            List<Integer> _columnIdxs = new ArrayList<Integer>();
            DataSet dataSet = DataSetFactory.newEmptyDataSet();
            for (int i = 0; i < header.length; i++) {
                String columnId = header[i];
                if (isColumnIncluded(columnId)) {
                    ColumnType type = ColumnType.LABEL;
                    if (firstRow != null) type = calculateType(columnId, firstRow[i]);
                    dataSet.addColumn(columnId, type);
                    _columnIdxs.add(i);
                }
            }

            // Load & insert the CSV rows
            if (firstRow != null) {
                Object[] _rowArray = new Object[dataSet.getColumns().size()];
                _processLine(dataSet, _rowArray, firstRow, _columnIdxs);
                dataSet.setValuesAt(dataSet.getRowCount(), _rowArray);
                String[] _line = csvReader.readNext();
                while (_line != null && _line.length == header.length) {
                    _processLine(dataSet,_rowArray, _line, _columnIdxs);
                    dataSet.setValuesAt(dataSet.getRowCount(), _rowArray);
                    _line = csvReader.readNext();
                }
            }
            return dataSet;
        } finally {
            is.close();
        }
    }

    protected InputStream getCSVInputStream() throws Exception {
        String url = dataSetDef.getFileURL();
        if (!StringUtils.isBlank(url)) {
            return new URL(url).openStream();
        }
        String path = dataSetDef.getFilePath();
        if (!StringUtils.isBlank(path)) {
            File f = new File(path);
            if (f.exists()) {
                return new FileInputStream(f);
            }
        }
        if (csvStorage != null) {
            InputStream is = csvStorage.getCSVInputStream(dataSetDef);
            if (is != null) {
                return is;
            }
        }
        throw new IllegalArgumentException("CSV content not found: " + dataSetDef);
    }

    protected ColumnType calculateType(String columnId, String value) {
        DataColumnDef column = dataSetDef.getColumnById(columnId);
        if (column != null) return column.getColumnType();

        try {
            DateFormat dateFormat = getDateFormat(columnId);
            dateFormat.parse(value);
            return ColumnType.DATE;
        } catch (Exception e) {
            try {
                DecimalFormat numberFormat = getNumberFormat(columnId);
                numberFormat.parse(value);
                return ColumnType.NUMBER;
            } catch (Exception ee) {
                return ColumnType.LABEL;
            }
        }
    }

    protected void _processLine(DataSet dataSet, Object[] row, String[] line, List<Integer> columnIdxs) throws Exception {
        List<DataColumn> columns = dataSet.getColumns();
        for (int i=0; i<columns.size(); i++) {
            DataColumn column = dataSet.getColumnByIndex(i);
            int columnIdx = columnIdxs.get(i);
            String valueStr = line[columnIdx];
            if (!StringUtils.isBlank(valueStr)){
                row[i] = parseValue(column, valueStr);
            } else {
                row[i] = null;
            }
        }
    }

    protected Object parseValue(DataColumn column, String value) throws Exception {
        ColumnType type = column.getColumnType();
        try {
            if (type.equals(ColumnType.DATE)) {
                String pattern = dataSetDef.getPattern(column.getId());
                // Handle special date pattern "epoch"
                if (pattern != null && DATE_FORMAT_EPOCH.equalsIgnoreCase(pattern)) {
                    Double _epoch = Double.parseDouble(value);
                    return new Date(_epoch.longValue() * 1000);
                } else {
                    DateFormat dateFormat = getDateFormat(column.getId());
                    return dateFormat.parse(value);
                }
            } else if (type.equals(ColumnType.NUMBER)) {
                DecimalFormat numberFormat = getNumberFormat(column.getId());
                return numberFormat.parse(value).doubleValue();
            } else {
                return value;
            }
        } catch (ParseException e) {
            String msg = "Error parsing value: " + value + ", " + e.getMessage() + ". Check column\u0027s data type consistency!";
            throw new Exception(msg);
        }
    }

    protected DateFormat getDateFormat(String columnId) {
        DateFormat format = _dateFormatMap.get(columnId);
        if (format == null) {
            format = new SimpleDateFormat(dataSetDef.getDatePattern(columnId));
            _dateFormatMap.put(columnId, format);
        }
        return format;
    }

    protected DecimalFormat getNumberFormat(String columnId) {
        DecimalFormat format = _numberFormatMap.get(columnId);
        if (format == null) {
            DecimalFormatSymbols numberSymbols = new DecimalFormatSymbols();
            numberSymbols.setGroupingSeparator(dataSetDef.getNumberGroupSeparator(columnId));
            numberSymbols.setDecimalSeparator(dataSetDef.getNumberDecimalSeparator(columnId));
            format = new DecimalFormat(dataSetDef.getNumberPattern(columnId), numberSymbols);
            _numberFormatMap.put(columnId, format);
        }
        return format;
    }
}