Java Examples for au.com.bytecode.opencsv.CSVReader

public static void main(String[] args) throws Exception {
    String[] row = null;
    URL stockURL = new URL("http://localhost:8888/tornadotweets.csv");
    BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
    CSVReader csvReader = new CSVReader(in);
    List content = csvReader.readAll();
    for (Object object : content) {
        row = (String[]) object;
        System.out.println(row[0] + " # " + row[1] + " #  " + row[2]);
    }
    csvReader.close();
}

@Test
public void testReadLineWithCommaSeparator() throws Exception {
    final BufferedReader reader = new BufferedReader(new FileReader(TEST_CSV));
    final CSVReader csvReader = new CSVReader(reader, '\t', '"');
    int res = 0;
    long time = System.currentTimeMillis();
    String[] line = null;
    while ((line = csvReader.readNext()) != null) {
        res += line.length;
    }
    time = System.currentTimeMillis() - time;
    System.out.println("time = " + time + " ms.");
    Assert.assertEquals(ROWS * COLS, res);
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

/**
     * This approach seems to work correctly, even with embedded newlines.
     *
     * @param originalCommentText
     * @throws FileNotFoundException
     * @throws IOException
     */
protected void testRawCsvRead(String originalCommentText) throws FileNotFoundException, IOException {
    CSVReader reader = new CSVReader(new FileReader(filePath));
    String[] nextLine = null;
    int count = 0;
    while ((nextLine = reader.readNext()) != null) {
        if (!nextLine[0].equals("field1")) {
            System.out.println("RawCsvRead Assert Error: Name is wrong.");
        }
        if (!nextLine[1].equals("3.0")) {
            System.out.println("RawCsvRead Assert Error: Value is wrong.");
        }
        if (!nextLine[2].equals("3,147.25")) {
            System.out.println("RawCsvRead Assert Error: Amount1 is wrong.");
        }
        if (!nextLine[3].equals("$3,147.26")) {
            System.out.println("RawCsvRead Assert Error: Currency is wrong.");
        }
        System.out.println("Field 4 read: " + nextLine[4]);
        if (!nextLine[4].equals(originalCommentText)) {
            System.out.println("RawCsvRead Assert Error: Comment is wrong.");
        }
        count++;
    }
    if (count != 3) {
        System.out.println("RawCsvRead Assert Error: Count of lines is wrong.");
    }
}

public List<SchemaCsv> read() {
    try {
        InputStream input = new FileInputStream(path);
        InputStreamReader ireader = new InputStreamReader(input, "UTF-8");
        CSVReader reader = new CSVReader(ireader, ',', '"', 1);
        ColumnPositionMappingStrategy<SchemaCsv> strat = new ColumnPositionMappingStrategy<SchemaCsv>();
        strat.setType(SchemaCsv.class);
        strat.setColumnMapping(SchemaCsv.COLUMNS);
        CsvToBean<SchemaCsv> csv = new CsvToBean<SchemaCsv>();
        return csv.parse(strat, reader);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

public static List<SoundEvent> read(String file, SoundEvent.Type type) throws IOException {
    List<SoundEvent> events = new ArrayList<SoundEvent>();
    CSVReader reader = new CSVReader(new FileReader(file));
    String[] nextLine;
    while ((nextLine = reader.readNext()) != null) {
        if (!isString(nextLine)) {
            SoundEvent e = new SoundEvent(nextLine[0], nextLine[1]);
            e.type = type;
            events.add(e);
        }
    }
    reader.close();
    return events;
}

@SuppressWarnings("unchecked")
private void loadRules() {
    if (src == null) {
        throw new SmooksException("ruleFile not specified.");
    }
    InputStream ruleStream;
    // Get the input stream...
    try {
        ruleStream = new URIResourceLocator().getResource(src);
    } catch (final IOException e) {
        throw new SmooksException("Failed to open rule file '" + src + "'.", e);
    }
    CSVReader csvLineReader = new CSVReader(new InputStreamReader(ruleStream));
    List<String[]> entries;
    try {
        entries = csvLineReader.readAll();
    } catch (IOException e) {
        throw new SmooksConfigurationException("Error reading MVEL rule file (CSV format) '" + src + "'.", e);
    } finally {
        try {
            ruleStream.close();
        } catch (IOException e) {
            logger.debug("Error closing MVEL rule file '" + src + "'.", e);
        }
    }
    for (String[] ruleLine : entries) {
        if (ruleLine.length == 2 && ruleLine[0].trim().charAt(0) != '#') {
            String ruleName = ruleLine[0].trim();
            String ruleExpression = ruleLine[1];
            if (rules.containsKey(ruleName)) {
                logger.debug("Duplicate rule definition '" + ruleName + "' in MVEL rule file '" + ruleName + "'.  Ignoring duplicate.");
                continue;
            }
            rules.put(ruleName, new MVELExpressionEvaluator().setExpression(ruleExpression));
        }
    }
}

@Override
protected Integer doInBackground(Bundle... params) {
    Bundle args = params[0];
    boolean erase = args.getBoolean(ImportActivity.WIPE_DATA);
    if (erase) {
        getParent().getContentResolver().delete(FillupsTable.BASE_URI, null, null);
        publishProgress(0, R.string.update_erased_database);
    }
    SimpleDateFormat formatter = new SimpleDateFormat(args.getString(CsvDateFormatActivity.DATE_FORMAT));
    getParent().getContentResolver().delete(CacheTable.BASE_URI, null, null);
    publishProgress(0, R.string.update_erased_cache);
    String base = args.getString(ImportActivity.FILENAME);
    String filename = Settings.EXTERNAL_DIR + base;
    CSVReader csvReader = null;
    int i = 0;
    try {
        BufferedReader reader = new BufferedReader(new FileReader(filename));
        csvReader = new CSVReader(reader);
        // skip the row of headers
        csvReader.readNext();
        String[] data;
        while ((data = csvReader.readNext()) != null) {
            try {
                ContentValues values = new ContentValues();
                setDouble(values, Fillup.TOTAL_COST, args, data);
                setDouble(values, Fillup.UNIT_PRICE, args, data);
                setDouble(values, Fillup.VOLUME, args, data);
                setDouble(values, Fillup.ODOMETER, args, data);
                setDouble(values, Fillup.ECONOMY, args, data);
                setDouble(values, Fillup.LATITUDE, args, data);
                setDouble(values, Fillup.LONGITUDE, args, data);
                setBoolean(values, Fillup.PARTIAL, args, data);
                setBoolean(values, Fillup.RESTART, args, data);
                int vehicleIndex = args.getInt(Fillup.VEHICLE_ID);
                String vehicle = data[vehicleIndex];
                long vehicleId = args.getLong("vehicle_" + vehicle);
                values.put(Fillup.VEHICLE_ID, vehicleId);
                int dateIndex = args.getInt(Fillup.DATE);
                String date = data[dateIndex];
                Date d = formatter.parse(date);
                values.put(Fillup.DATE, d.getTime());
                Fillup f = new Fillup(values);
                f.save(getParent());
                publishProgress(++i);
            } catch (InvalidFieldException e) {
                publishProgress(++i, e.getErrorMessage());
            }
        }
    } catch (IOException e) {
    } catch (ParseException e) {
        Log.e(TAG, "Couldn't parse a field!", e);
    } finally {
        try {
            if (csvReader != null) {
                csvReader.close();
            }
        } catch (IOException e2) {
        }
    }
    return i;
}

@Override
public void execute(Tuple input) {
    try {
        LocationBoundaryFileDto locationBoundaryFileDto = (LocationBoundaryFileDto) input.getValueByField("locationBoundaryFileDto");
        CSVReader csvReader = new CSVReader(new FileReader(locationBoundaryFileDto.getFileNameAndPath()));
        String headers[] = csvReader.readNext();
        //First column will be latitude and second column will be longitude
        String line[];
        while ((line = csvReader.readNext()) != null) {
        }
    } catch (Exception ex) {
        collector.fail(input);
    }
}

public String[] getNextRow() throws IOException {
    String[] line = csvReader.readNext();
    if (line == null) {
        fileIndex++;
        if (fileIndex < fileArray.length) {
            csvReader.close();
            csvReader = new CSVReader(getReaderForFileType(fileArray[fileIndex]));
            return getNextRow();
        }
        return null;
    }
    return line;
}

//------------------------ PRIVATE --------------------------
private List<Set<String>> createAltNamesDictionaryFromCsv(String csvClasspath) throws IOException {
    List<Set<String>> dictionary = Lists.newArrayList();
    try (CSVReader reader = new CSVReader(new InputStreamReader(this.getClass().getResourceAsStream(csvClasspath), "UTF-8"))) {
        String[] next = reader.readNext();
        while (next != null) {
            dictionary.add(Sets.newHashSet(next));
            next = reader.readNext();
        }
    }
    return dictionary;
}

public static Map<String, String> getPluginPropertyMap(String propertyString) throws IOException {
    Map propertyMap = new HashMap();
    if (propertyString != null && propertyString.trim().length() > 0) {
        CSVReader reader = new CSVReader(new StringReader(propertyString));
        List entries = reader.readAll();
        for (int i = 0; i < entries.size(); i++) {
            String[] entry = (String[]) entries.get(i);
            propertyMap.put(entry[0], entry[1]);
        }
    }
    return propertyMap;
}

public static List<Record> parseCreationFile(String path, char delimiter) throws IOException {
    ColumnPositionMappingStrategy<Record> strategy = new ColumnPositionMappingStrategy<Record>();
    strategy.setType(Record.class);
    String[] columns = new String[] { "category", "name", "template", "url" };
    strategy.setColumnMapping(columns);
    CSVReader reader = new CSVReader(new FileReader(path), delimiter, '\"', 1);
    CsvToBean<Record> csv = new CsvToBean<Record>();
    return csv.parse(strategy, reader);
}

@Override
protected Object doTransform(final Object src, final String enc) throws TransformerException {
    List<Map> products = new ArrayList<Map>();
    CSVReader reader = new CSVReader(new StringReader((String) src));
    String[] nextLine;
    try {
        while ((nextLine = reader.readNext()) != null) {
            Map<String, String> product = new HashMap<String, String>();
            product.put("name", nextLine[0]);
            product.put("acv", nextLine[1]);
            product.put("cost", nextLine[2]);
            product.put("description", nextLine[3]);
            products.add(product);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    return products;
}

@Override
public CSVReader getInput() throws IOException {
    char fieldSep = config.getFieldSeparator() == null ? CSVParser.DEFAULT_SEPARATOR : config.getFieldSeparator();
    char quoteChar = config.getFieldDelimiter() == null ? CSVParser.DEFAULT_QUOTE_CHARACTER : config.getFieldDelimiter();
    char escapeChar = config.getEscapeCharacter() == null ? CSVParser.DEFAULT_ESCAPE_CHARACTER : config.getEscapeCharacter();
    Reader reader = readerSupplier.getInput();
    CSVReader result = new CSVReader(reader, fieldSep, quoteChar, escapeChar, 0, false);
    return result;
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

@Test
public void testOpenCSV() throws IOException {
    final LhaFile lhaFile = new LhaFile(new File("src/test/resources/", "ken_all.lzh"));
    try {
        final LhaHeader[] entries = lhaFile.getEntries();
        for (final LhaHeader entry : entries) {
            final au.com.bytecode.opencsv.CSVReader reader = new au.com.bytecode.opencsv.CSVReader(new InputStreamReader(lhaFile.getInputStream(entry), "Windows-31J"), ',', '"');
            try {
                String[] values;
                while ((values = reader.readNext()) != null) {
                    continue;
                }
            } finally {
                reader.close();
            }
        }
    } finally {
        lhaFile.close();
    }
}

public void importDataFromCSV(String filepath) throws IOException {
    ClassLoader classLoader = this.getClass().getClassLoader();
    if (classLoader == null) {
        //$NON-NLS-1$
        throw new IllegalStateException("Cannot import data - null classloader");
    }
    //$NON-NLS-1$
    Reader fileReader = new InputStreamReader(classLoader.getResourceAsStream(filepath), Charset.forName("UTF-8"));
    try (CSVReader csvFile = new CSVReader(fileReader)) {
        final List<BenefitInfo> benefitsInfo = readBenefitsInfo(csvFile);
        persistBenefitsInfo(benefitsInfo);
    }
}

List<Object> readBeans() throws IOException {
    final CGLibMappingStrategy strategy = new CGLibMappingStrategy();
    final CsvToBean<Object> csvToBean = new CsvToBean<>();
    try (final CSVReader reader = new CSVReader(Files.newBufferedReader(metadataFile, charset), separator, quote, escape, skipLines, strictQuotes, ignoreLeadingWhiteSpace)) {
        return csvToBean.parse(strategy, reader);
    }
}

public JSONObject preprocessData(JSONObject parameterObject) {
    System.out.println("Preprocessing the input data...");
    try {
        double duplicateDistance = parameterObject.getDouble("eps");
        String inputFilePath = parameterObject.getString("dataPath");
        File inputFile = new File(inputFilePath);
        FileReader inputFileReader = new FileReader(inputFile);
        CSVReader inputCSVReader = new CSVReader(inputFileReader);
        String newLineSymbol = System.getProperty("line.separator");
        String tempFileName = parameterObject.getString("tempPath") + "/" + inputFile.getName().replace(".csv", "") + "_processed.csv";
        File outputFile = new File(tempFileName);
        if (outputFile.exists()) {
            outputFile.delete();
            outputFile.createNewFile();
        }
        FileWriter outputFileWriter = new FileWriter(outputFile, true);
        // get the indexes
        int recordIDIndex = parameterObject.getInt("recordIDIndex");
        int userIDIndex = parameterObject.getInt("userIDIndex");
        int lngIndex = parameterObject.getInt("lngIndex");
        int latIndex = parameterObject.getInt("latIndex");
        /*boolean removeDuplicates = parameterObject.getBoolean("removeDuplicates");
						if(removeDuplicates && (userIDIndex == -1))
						{
								System.out.println("You have indicated to remove duplicated records, but didn't provide the column number of user id; please check the configuration file.");
								inputCSVReader.close();
								outputFileWriter.close();
								return null;
						}*/
        Hashtable<String, Vector<Point2D>> existingDataHashtable = new Hashtable<>(1000);
        long totalProcessedDataRecord = 0;
        String[] thisInputLine = inputCSVReader.readNext();
        while ((thisInputLine = inputCSVReader.readNext()) != null) {
            String recordId = null;
            String ownerString = null;
            double latString = 0;
            double lngString = 0;
            try {
                recordId = thisInputLine[recordIDIndex];
                if (userIDIndex != -1)
                    ownerString = thisInputLine[userIDIndex];
                latString = Double.parseDouble(thisInputLine[latIndex]);
                lngString = Double.parseDouble(thisInputLine[lngIndex]);
            } catch (Exception e) {
                continue;
            }
            if (userIDIndex != -1) {
                boolean isDuplicated = false;
                if (existingDataHashtable.containsKey(ownerString)) {
                    Vector<Point2D> peopleCoordsVector = existingDataHashtable.get(ownerString);
                    Iterator<Point2D> coordIterator = peopleCoordsVector.iterator();
                    while (coordIterator.hasNext()) {
                        Point2D coordPoint = coordIterator.next();
                        double thisDistance = Math.sqrt((coordPoint.getX() - lngString) * (coordPoint.getX() - lngString) + (coordPoint.getY() - latString) * (coordPoint.getY() - latString));
                        if (thisDistance <= duplicateDistance) {
                            isDuplicated = true;
                            break;
                        }
                    }
                } else {
                    Vector<Point2D> peopleCoordsVector = new Vector<>(10);
                    existingDataHashtable.put(ownerString, peopleCoordsVector);
                }
                if (!isDuplicated) {
                    // add this point to existing table       									
                    Vector<Point2D> peopleCoordsVector = existingDataHashtable.get(ownerString);
                    peopleCoordsVector.add(new Point2D(lngString, latString));
                    existingDataHashtable.put(ownerString, peopleCoordsVector);
                    outputFileWriter.append(recordId + "," + ownerString + "," + latString + "," + lngString + newLineSymbol);
                    totalProcessedDataRecord++;
                }
            } else {
                /*if(userIDIndex != -1)
										{
												existingDataHashtable.put(ownerString, new Vector<Point2D>(1));
												outputFileWriter.append(recordId+","+ownerString+","+latString+","+lngString+newLineSymbol);
										}
										else
										{*/
                outputFileWriter.append(recordId + "," + latString + "," + lngString + newLineSymbol);
                //}
                totalProcessedDataRecord++;
            }
        }
        inputCSVReader.close();
        outputFileWriter.close();
        JSONObject resultObject = new JSONObject();
        resultObject.put("file", tempFileName);
        if (userIDIndex != -1)
            resultObject.put("userCount", existingDataHashtable.size());
        resultObject.put("recordCount", totalProcessedDataRecord);
        if (userIDIndex != -1)
            System.out.println("After pre-processing, there are " + totalProcessedDataRecord + " records and " + existingDataHashtable.size() + " users in the data.");
        else
            System.out.println("After pre-processing, there are " + totalProcessedDataRecord + " records in the data.");
        System.out.println("---------------------------------------------------");
        return resultObject;
    } catch (Exception e) {
        System.out.println("An error happened in the data preprocessing; The program has been canceled");
        return null;
    }
}

@Override
public IStatus doImport(IProgressMonitor monitor) throws Exception {
    CSVReader reader = new CSVReader(new FileReader(results[0]), ';');
    monitor.beginTask("Importiere Physio", 100);
    String[] line = reader.readNext();
    while ((line = reader.readNext()) != null) {
        if (line.length < 3) {
            continue;
        }
        monitor.subTask(line[1]);
        String id = new Query<PhysioLeistung>(PhysioLeistung.class).findSingle("Ziffer", "=", line[0]);
        if (id != null) {
            PhysioLeistung pl = PhysioLeistung.load(id);
            pl.set(new String[] { "Titel", "TP" }, line[1], line[2]);
        } else {
            /* PhysioLeistung pl = */
            new PhysioLeistung(line[0], line[1], line[2], null, null);
        }
    }
    monitor.done();
    return Status.OK_STATUS;
}

public static SortedMap<Number, Number> importCsv(String filePath) {
    try {
        reader = new CSVReader(new FileReader(filePath));
        List<String[]> entries = reader.readAll();
        SortedMap<Number, Number> data = new TreeMap<>();
        for (String[] line : entries) {
            if (line.length == 2) {
                Number x = NumberFormat.getNumberInstance(LOCALE).parse(line[0]);
                Number y = NumberFormat.getNumberInstance(LOCALE).parse(line[1]);
                data.put(x, y);
            }
        }
        reader.close();
        return data;
    } catch (IOExceptionParseException |  e) {
        System.err.println(e.getMessage());
    }
    return Collections.emptySortedMap();
}

/**
	 * Reads a csv lookup table file. The selected columns specified by
	 * parameters keyColumn and valueColumn are mapped together.
	 * 
	 * @param input the inputstream of the csv file
	 * @param charset specific charset of the csv file
	 * @param separator used separator char in csv file
	 * @param quote used quote char in csv file
	 * @param escape used escape char in csv file
	 * @param skipFirst true, if first line should be skipped
	 * @param keyColumn source column of the lookup table
	 * @param valueColumn target column of the lookup table
	 * @return lookup table as map
	 * @throws IOException if inputstream is not readable
	 */
public Map<Value, Value> read(InputStream input, Charset charset, char separator, char quote, char escape, boolean skipFirst, int keyColumn, int valueColumn) throws IOException {
    Reader streamReader = new BufferedReader(new InputStreamReader(input, charset));
    CSVReader reader = new CSVReader(streamReader, separator, quote, escape);
    String[] nextLine;
    Map<Value, Value> values = new LinkedHashMap<Value, Value>();
    if (skipFirst)
        reader.readNext();
    while ((nextLine = reader.readNext()) != null) {
        if (nextLine.length >= 2)
            values.put(Value.of(nextLine[keyColumn]), Value.of(nextLine[valueColumn]));
    }
    reader.close();
    return values;
}

protected void internalProcess() throws Exception {
    final URL csvFile = GogreenKonakartSynchronizationService.class.getClassLoader().getResource(filePath);
    if (csvFile != null) {
        final InputStream csvIS = csvFile.openStream();
        final InputStreamReader csvISR = createInputStream(csvIS);
        String[] csvLine;
        final CSVReader csvReader = new CSVReader(csvISR, ',', '"', '~', 1);
        int i = 0;
        while ((csvLine = csvReader.readNext()) != null) {
            System.out.println("process the line: " + i);
            processRow(csvLine);
            i++;
        }
    }
}

@Test
public void testCreateStudentObjectsFromCSV() {
    CSVReader csvReader;
    try {
        String encoding = checkEncoding("./src/test/java/de/dhbw/humbuch/util/schueler_stamm.csv");
        if (encoding != null) {
            csvReader = new CSVReader(new InputStreamReader(new FileInputStream("./src/test/java/de/dhbw/humbuch/util/schueler_stamm.csv"), encoding), ';', '\'', 0);
        } else {
            csvReader = new CSVReader(new FileReader("./src/test/java/de/dhbw/humbuch/util/schueler_stamm.csv"), ';', '\'', 0);
        }
        ArrayList<de.dhbw.humbuch.model.entity.Student> list = CSVHandler.createStudentObjectsFromCSV(csvReader);
        assertEquals(99, list.size());
        assertEquals("Zivko", list.get(1).getLastname());
        assertEquals("Adelina", list.get(1).getFirstname());
        DateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy");
        assertEquals("02.01.1989", dateFormat.format(list.get(1).getBirthday()));
        assertEquals("m", list.get(1).getGender());
    } catch (FileNotFoundExceptionUnsupportedEncodingException |  e) {
        e.printStackTrace();
    }
}

/**
     * Retrieve plugin properties from a CSV string
     * 
     * @deprecated method used by Joget v2 to parse plugin properties from a CSV string. Since Joget v3,
     * Joget introduced a better UI for plugin configuration, the properties are store in JSON format.
     * 
     * @param propertyString
     * @return
     * @throws IOException 
     */
public static Map<String, String> getPluginPropertyMap(String propertyString) throws IOException {
    Map propertyMap = new HashMap();
    if (propertyString != null && propertyString.trim().length() > 0) {
        CSVReader reader = new CSVReader(new StringReader(propertyString));
        List entries = reader.readAll();
        for (int i = 0; i < entries.size(); i++) {
            String[] entry = (String[]) entries.get(i);
            propertyMap.put(entry[0], entry[1]);
        }
    }
    return propertyMap;
}

public void readJournal() throws IOException {
    CSVReader csvReader = new CSVReader(new FileReader(file));
    try {
        for (String[] row = csvReader.readNext(); row != null; row = csvReader.readNext()) {
            KanbanJournalItem item = new KanbanJournalItem(Integer.parseInt(row[0]), row[1], row[2], row[3]);
            journalItems.add(item);
        }
        //        journalItems.add(new KanbanJournalItem("2011-10-10", "Hello", "Robert"));
        //        journalItems.add(new KanbanJournalItem("2011-12-10", "World", "Rob"));
        Collections.sort(journalItems);
    } finally {
        csvReader.close();
    }
}

public List<PersistedObject> getPersistedObjects() {
    List<PersistedObject> persistedObjects = new ArrayList<>();
    try {
        CSVReader reader = new CSVReader(new FileReader(NameFactory.PERSISTENCE_CSV));
        List<String[]> allEntries = reader.readAll();
        reader.close();
        for (String[] entry : allEntries) {
            persistedObjects.add(new PersistedObject(entry[1], entry[0]));
        }
    } catch (Exception e) {
        App.kill(e);
    }
    return persistedObjects;
}

@Test
public void read_CSV_File_With_OpenCSV() {
    // If you are using spring, you could use a bit cleaner way Resource resource = new ClassPathResource
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("planets.csv");
    try {
        CSVReader reader = new CSVReader(new InputStreamReader(in), '\t');
        List<String[]> rows = reader.readAll();
        for (int x = 0; x < rows.size(); x++) {
            String[] columns = rows.get(x);
            for (int y = 0; y < columns.length; y++) {
                logger.info(columns[y]);
            }
        }
        reader.close();
    } catch (FileNotFoundException e) {
        logger.error(e);
    } catch (IOException e) {
        logger.error(e);
    }
}

/**
     * 
     * @param <T>
     * @param path
     * @param delimiter
     * @param builder
     * @return
     * @throws Exception 
     */
public static <T> List<T> getAll(final String path, final char delimiter, final Builder<T> builder) throws Exception {
    try {
        final List<T> result = new LinkedList<T>();
        final FileReader fr = new FileReader(path);
        final CSVReader reader = new CSVReader(fr, delimiter);
        String[] nextLine;
        while ((nextLine = reader.readNext()) != null) {
            result.add(builder.newInstance(nextLine));
        }
        return result;
    } catch (Exception ex) {
        throw new Exception("Error while reading CSV file!", ex);
    }
}

public static Map<String, ManageableHoliday> load(final String resourceLocation, final String regionScheme) {
    final ResourceBundle holidayProperties = ResourceBundle.getBundle(resourceLocation);
    final Map<String, ManageableHoliday> holidays = Maps.newHashMapWithExpectedSize(holidayProperties.keySet().size());
    CSVReader csvReader;
    for (final String regionCode : holidayProperties.keySet()) {
        final String file = holidayProperties.getString(regionCode);
        ManageableHoliday holiday = holidays.get(regionCode);
        if (holiday == null) {
            //// old style with region
            holiday = new ManageableHoliday();
            //holiday.setType(HolidayType.BANK);
            //holiday.setRegionExternalId(ExternalId.of(regionScheme, regionCode));
            //holidays.put(regionCode, holiday);
            holiday.setType(HolidayType.CUSTOM);
            holiday.setCustomExternalId(ExternalId.of(regionScheme, regionCode));
            holidays.put(regionCode, holiday);
        }
        if (file.trim().isEmpty()) {
            // no holidays, will use standard weekday calendar
            continue;
        }
        try {
            final File filepath = ResourceUtils.createResource(file).getFile();
            csvReader = new CSVReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(filepath))));
        } catch (FileNotFoundException ex) {
            throw new OpenGammaRuntimeException("file not found: " + file);
        } catch (IOException ex) {
            throw new OpenGammaRuntimeException("IO Exception: " + ex);
        }
        String[] currLine;
        //csvReader.readNext();
        try {
            while ((currLine = csvReader.readNext()) != null) {
                String dateInUSFormat = currLine[0].trim();
                if (dateInUSFormat.startsWith("#")) {
                    continue;
                }
                LocalDate date = LocalDate.parse(dateInUSFormat, US_FORMATTER);
                holiday.getHolidayDates().add(date);
            }
        } catch (IOException ex) {
            throw new OpenGammaRuntimeException("IOError: " + ex);
        } finally {
            try {
                csvReader.close();
            } catch (IOException ex) {
                throw new OpenGammaRuntimeException("IOError on closing: " + ex);
            }
        }
    }
    return holidays;
}

private List<GroundTruthBean> readGroundTruths(InputStream groundTruthCsv) throws IOException {
    CSVReader reader = new CSVReader(new InputStreamReader(groundTruthCsv), '\t');
    ColumnPositionMappingStrategy<GroundTruthBean> strat = new ColumnPositionMappingStrategy<GroundTruthBean>();
    strat.setColumnMapping(new String[] { "accuracy", "mime", "charset", "digest", "extensions", "filename", "id", "kind", "size", "version" });
    strat.setType(GroundTruthBean.class);
    CsvToBean<GroundTruthBean> csv = new CsvToBean<GroundTruthBean>();
    return csv.parse(strat, reader);
}

private TicketProvider parseFile() throws IOException, InterruptedException {
    // Tab delimited reader with no un-escaping of '\' characters.
    CSVReader csvReader = new CSVReader(reader, '\t', '"', '\0');
    try {
        String[] headings = csvReader.readNext();
        if (headings == null) {
            throw new IOException("Empty input given");
        }
        final TracTabResult result;
        try {
            result = new TracTabResult(headings);
        } catch (RuntimeException ex) {
            throw new IOException(ex);
        }
        String[] fields;
        while ((fields = csvReader.readNext()) != null) {
            if (Thread.currentThread().isInterrupted()) {
                throw new InterruptedException();
            }
            result.addTicketFromFields(fields);
        }
        return result;
    } finally {
        csvReader.close();
    }
}

public static void main(String[] args) throws IOException {
    Preconditions.checkArgument(args.length == 2, "Usage: wn30msa-to-wn31 path/to/wn-msa-all.tab dest/dir");
    File wn30coreFile = new File(args[0]);
    File wn31TsvFile = new File(args[1], "wn31-msa-all.tab");
    log.info("Loading wn30-msa '{}' (and converting to wn31-msa '{}')...", wn30coreFile, wn31TsvFile);
    int index = 1;
    try (CSVWriter writer = new CSVWriter(new FileWriter(wn31TsvFile), '\t', CSVWriter.NO_QUOTE_CHARACTER)) {
        try (CSVReader reader = new CSVReader(new FileReader(wn30coreFile), '\t')) {
            while (true) {
                String[] row = reader.readNext();
                if (row == null) {
                    break;
                }
                String wn30sense = row[0];
                char posLetter = wn30sense.charAt(9);
                final Integer posNumeric = Preconditions.checkNotNull(Wn30ToWn31.POS_NUMERIC.get(posLetter), "Invalid part-of-speech letter code '%s' for sense '%s'", posLetter, wn30sense);
                String wn31sense = posNumeric + wn30sense;
                writer.writeNext(new String[] { wn31sense, row[1], row[2], row[3] });
                index++;
            }
        }
    }
    log.info("Converted {} rows", index);
}

public List<BasicData> loadSunspots() throws IOException {
    // Obtain file
    final InputStream istream = this.getClass().getResourceAsStream("/sunspots.csv");
    if (istream == null) {
        System.out.println("Cannot access data set, make sure the resources are available.");
        System.exit(1);
    }
    final Reader reader = new InputStreamReader(istream);
    final CSVReader csv = new CSVReader(reader);
    csv.readNext();
    // determine how many entries in file
    int sunspotCount = 0;
    while (csv.readNext() != null) {
        sunspotCount++;
    }
    System.out.println("Sunspot count:" + sunspotCount);
    // allocate array to hold file
    double[][] dataset = new double[sunspotCount][1];
    // read file
    final InputStream istream2 = this.getClass().getResourceAsStream("/sunspots.csv");
    final Reader reader2 = new InputStreamReader(istream2);
    final CSVReader csv2 = new CSVReader(reader2);
    csv2.readNext();
    String[] nextLine;
    int idx = 0;
    while ((nextLine = csv2.readNext()) != null) {
        double ssn = Double.parseDouble(nextLine[3]);
        dataset[idx++][0] = ssn;
    }
    // timseries encode
    List<BasicData> result = TimeSeriesUtil.slidingWindow(dataset, this.INPUT_WINDOW, 1, new int[] { 0 }, new int[] { 0 });
    return result;
}

@Override
public <T> List<T> parse(Reader csv, IParseStrategy<T> strategy) throws CsvException {
    List<T> ret = new LinkedList<>();
    try (CSVReader reader = new CSVReader(csv, strategy.getSeparator(), strategy.getQuote(), strategy.getEscape(), strategy.getSkipLines(), strategy.getStrictQuotes(), strategy.getIgnoreLeadingWhiteSpace())) {
        IDataConverter<T> converter = strategy.getConverter();
        String[] line;
        while ((line = reader.readNext()) != null) {
            ret.add(converter.convertToData(line));
        }
    } catch (IOException e) {
        throw new CsvException(e);
    }
    return ret;
}

/**
	 * Imports an csv-based file and a belonging .ssd-file.
	 * 
	 * @param input
	 *            Reference of the file which should be imported.
	 * @param algoOut
	 *            Reference of the .ssd file which is made by a Datamining algorithm.
	 * 
	 * @throws IOException
	 *             threw if something other is going wrong.
	 * @throws DatabaseAccessException
	 *             threw if something went wrong with the Database connection.
	 * @throws InvalidFileException
	 *             threw if importing file isn't valid.
	 */
@Override
public void importFile(final File input, final File algoOut) throws IOException, DatabaseAccessException, InvalidFileException {
    if (input == null || algoOut == null || !input.exists() || !algoOut.exists()) {
        throw new FileNotFoundException();
    }
    CSVFileInfo info = (CSVFileInfo) this.getFileInfoExtractor().extractFileInfo(input);
    CSVReader cr = new CSVReader(new FileReader(input), info.getDelimiter(), '\'', info.getFirstLineOfDataSegment() - 1);
    if (info.getFeatures().length < 2) {
        throw new InvalidFileException();
    }
    //amount of queues is equal to the amount of defined subspaces.
    LinkedBlockingQueue<Float>[] queues = this.parseAlgoOut(algoOut, info.noOfFeatures());
    //number of all features (natural features + outlierness values.
    int amountFeatures = queues.length + info.noOfFeatures();
    int infoNoOfFeatures = info.noOfFeatures();
    //because the class array is no real feature.
    if (info.getFeatures()[info.getFeatures().length - 1].matches("class(.)*")) {
        --amountFeatures;
        --infoNoOfFeatures;
    }
    //amount of all features which are detected in ssd.
    int amountObjects = queues[0].size();
    String[] allFeatures = new String[amountFeatures];
    boolean[] featureTypes = new boolean[amountFeatures];
    int i = 0;
    for (; i < infoNoOfFeatures; ++i) {
        allFeatures[i] = info.getFeatures()[i];
        featureTypes[i] = false;
    }
    for (int k = 0; i < amountFeatures; ++k, ++i) {
        allFeatures[i] = "Outlierness" + (k + 1);
        featureTypes[i] = true;
    }
    getDB().initFeatures(allFeatures, featureTypes);
    String[] line = null;
    //buffervector of build objects which will be stored next.
    float[][] batch = new float[BATCH_SIZE][];
    //current built object.
    float[] actObj = null;
    int actBatchSize = 0;
    for (int k = 0; k < amountObjects; ++k) {
        int pos = 0;
        actObj = new float[amountFeatures];
        line = cr.readNext();
        if (line.length == 0 || (line.length == 1 & (line[0] == null || line[0].isEmpty())) || line.length == 2 & (line[0] == null || line[0].isEmpty()) & (line[1] == null || line[1].isEmpty())) {
            --k;
            continue;
        }
        for (; pos < infoNoOfFeatures; ++pos) {
            try {
                actObj[pos] = Float.parseFloat(line[pos]);
            } catch (NumberFormatException e) {
                actObj[pos] = Float.NaN;
            }
        }
        for (int m = 0; m < queues.length; ++m, ++pos) {
            actObj[pos] = queues[m].poll();
        }
        batch[actBatchSize++] = actObj;
        if (actBatchSize >= BATCH_SIZE) {
            getDB().pushObject(batch);
            actBatchSize = 0;
            batch = new float[BATCH_SIZE][];
            System.gc();
        }
    }
    // removing null vectors from last batch matrix.
    int noVectors = 0;
    while (noVectors < BATCH_SIZE && batch[noVectors] != null) {
        noVectors++;
    }
    float[][] batchOle = new float[noVectors][amountFeatures];
    for (int k = 0; k < noVectors; ++k) {
        batchOle[k] = batch[k];
    }
    // push last batch matrix.
    this.getDB().pushObject(batchOle);
    // update db min/max values
    this.getDB().updateFeaturesMinMax();
    cr.close();
}

@Override
public FixTable create(TestRun testRun) throws Exception {
    try (CSVReader reader = new CSVReader(getFileReader(testRun), separator, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER)) {
        List<String[]> myEntries = reader.readAll();
        String[] colNames = myEntries.get(0);
        List<String[]> rowData = myEntries.subList(1, myEntries.size());
        return StringArrayFixTableCreator.createFixTable(colNames, rowData);
    }
}

private static ODLDatastoreAlterable<ODLTableAlterable> importFile(CSVReader reader, String tableName) {
    try {
        List<String[]> list = reader.readAll();
        int n = list.size();
        if (n > 0) {
            ODLDatastoreAlterable<ODLTableAlterable> ret = ODLDatastoreImpl.alterableFactory.create();
            ODLTableAlterable table = ret.createTable(tableName, -1);
            // create all columns
            for (String col : list.get(0)) {
                // give default name if invalid
                if (col == null || Strings.isEmptyWhenStandardised(col) || TableUtils.findColumnIndx(table, col, true) != -1) {
                    col = TableUtils.getUniqueNumberedColumnName(DEFAULT_COLUMN_NAME, table);
                }
                table.addColumn(-1, col, ODLColumnType.STRING, 0);
            }
            for (int line = 1; line < n; line++) {
                String[] row = list.get(line);
                if (row.length != table.getColumnCount()) {
                    throw new RuntimeException("Line found with different number of columns to header line: " + line);
                }
                int rowIndx = table.createEmptyRow(line);
                for (int col = 0; col < row.length; col++) {
                    table.setValueAt(row[col], rowIndx, col);
                }
            }
            return ret;
        }
        return null;
    } catch (Throwable e) {
        throw new RuntimeException(e);
    }
}

/**
     * Return a List with the contents of a given CSV with supplied separator and quote char.
     *
     * @param fileName The path to the CSV to parse
     * @param splitBy The delimiter to use for separating entries
     * @param quoteChar The character to use for quoted elements
     * @param startFromLine The line number to skip for start reading
     */
public static List<String[]> loadData(String fileName, char splitBy, char quoteChar, int startFromLine) {
    List<String[]> contents = new ArrayList<>();
    try {
        CSVReader reader = new CSVReader(new FileReader(fileName), splitBy, quoteChar, startFromLine);
        String[] nextLine;
        int counter = 0;
        while ((nextLine = reader.readNext()) != null) {
            if (nextLine != null && nextLine.length > 0) {
                try {
                    contents.add(nextLine);
                //contents.put(counter, nextLine);
                //counter++;
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
        }
    } catch (Exception ex) {
        ex.printStackTrace();
    }
    return contents;
}

public static List<GraphMethod<?>> parseCsv(PropertyGraph graph, GraphLoadTestStats stats, File csvFile) throws IOException {
    CSVReader reader = null;
    List<GraphMethod<?>> methods = new ArrayList<GraphMethod<?>>();
    try {
        reader = new CSVReader(new FileReader(csvFile));
        String[] row;
        while ((row = reader.readNext()) != null) {
            methods.add(parseCsvRow(graph, stats, row));
        }
    } finally {
        IOUtils.closeQuietly(reader);
    }
    return methods;
}

public void testCSVReader() {
    // first create a FilePath to load the test Properties file.
    File workspaceDirFile = new File("target/test-classes/");
    FilePath workspaceRootDir = new FilePath(workspaceDirFile);
    LOGGER.info("workspace File path: " + workspaceDirFile.getAbsolutePath());
    LOGGER.info("workspace Dir path: " + workspaceRootDir.getName());
    CSVReader csvreader = null;
    InputStream in = null;
    InputStreamReader inputReader = null;
    FilePath[] seriesFiles = null;
    try {
        seriesFiles = workspaceRootDir.list(files[0]);
        if (seriesFiles != null && seriesFiles.length < 1) {
            LOGGER.info("No plot data file found: " + workspaceRootDir.getName() + " " + files[0]);
            assertFalse(true);
        }
        LOGGER.info("Loading plot series data from: " + files[0]);
        in = seriesFiles[0].read();
        inputReader = new InputStreamReader(in);
        csvreader = new CSVReader(inputReader);
        // save the header line to use it for the plot labels.
        String[] nextLine;
        // read each line of the CSV file and add to rawPlotData
        int lineNum = 0;
        while ((nextLine = csvreader.readNext()) != null) {
            // for some reason csvreader returns an empty line sometimes.
            if (nextLine.length == 1 && nextLine[0].length() == 0)
                break;
            if (columns[0] != nextLine.length) {
                StringBuilder msg = new StringBuilder();
                msg.append("column count is not equal ").append(nextLine.length);
                msg.append(" expected ").append(columns[0]).append(" at line ");
                msg.append(lineNum).append(" line: ").append("'");
                for (String s : nextLine) {
                    msg.append("\"").append(s).append("\":").append(s.length()).append(",");
                }
                msg.append("' length ").append(nextLine.length);
                assertTrue(msg.toString(), columns[0] == nextLine.length);
            }
            ++lineNum;
        }
        assertTrue("Line count is not equal " + lineNum + " expected " + lines[0], lines[0] == lineNum);
    } catch (IOException e) {
        assertFalse("Exception " + e, true);
    } catch (InterruptedException e) {
        assertFalse("Exception " + e, true);
    } finally {
        try {
            if (csvreader != null)
                csvreader.close();
        } catch (IOException e) {
        }
        try {
            if (inputReader != null)
                inputReader.close();
        } catch (IOException e) {
        }
        try {
            if (in != null)
                in.close();
        } catch (IOException e) {
        }
    }
}

private void readCsvFile(String filePath) {
    FileInputStream is = null;
    CSVReader reader = null;
    long p = 0;
    long count = 0;
    try {
        File f = new File(filePath);
        is = new FileInputStream(f);
        reader = new CSVReader(new InputStreamReader(is, cs), ',', '\"', '\0');
        String[] headers = reader.readNext();
        int headerCount = headers.length;
        while (true) {
            String[] items = reader.readNext();
            if (items == null)
                break;
            p++;
            if (p <= offset)
                continue;
            if (limit != 0 && count >= limit)
                break;
            int itemCount = items.length;
            Map<String, Object> m = new HashMap<String, Object>();
            for (int i = 0; i < Math.min(headerCount, itemCount); i++) {
                m.put(headers[i], items[i]);
            }
            if (itemCount > headerCount) {
                for (int i = headerCount; i < itemCount; i++) {
                    m.put("column" + i, items[i]);
                }
            }
            m.put("_file", f.getName());
            pushPipe(new Row(m));
            count++;
        }
    } catch (Throwable t) {
        throw new RuntimeException("csvfile load failure", t);
    } finally {
        IoHelper.close(reader);
        IoHelper.close(is);
    }
}

public synchronized HashMap<String, List<String>> loadDictionary(String fileName) throws IOException {
    // it's already loaded, just return it
    if (dictMap.containsKey(fileName)) {
        return dictMap.get(fileName);
    }
    // It's not loaded, load the file and put it in the dict map and return.
    // Assume the file is a csv file with key/value pairs on each line.
    HashMap<String, List<String>> dictionary = new HashMap<String, List<String>>();
    File dictFile = new File(fileName);
    if (!dictFile.exists()) {
        log.warn("Dictionary file not found {}", dictFile.getAbsolutePath());
        return null;
    }
    CSVReader reader = new CSVReader(new FileReader(fileName));
    String[] line;
    while ((line = reader.readNext()) != null) {
        // line[] is an array of values from the line
        List<String> listVal = dictionary.get(line[0]);
        if (listVal == null) {
            listVal = new ArrayList<String>();
            dictionary.put(line[0], listVal);
        }
        for (int i = 1; i < line.length; i++) {
            listVal.add(line[i]);
        }
    }
    dictMap.put(fileName, dictionary);
    reader.close();
    return dictionary;
}

public List<Object> importDataFromStream(InputStream stream, EntityImportStrategy entityStrategy, Long loggedUserId, Locale locale) throws UnsupportedEncodingException {
    List<Object> list = new ArrayList<>();
    CSVReader reader = new CSVReader(new InputStreamReader(stream, "UTF-8"));
    try {
        String[] firstLine = reader.readNext();
        this.headerFields = firstLine;
        String[] nextLine;
        int rowNum = 1;
        EntityHandlingStrategy entityHandler = DataImportStrategyProvider.instance().getEntityHandler(entityStrategy);
        if (firstLine != null) {
            while ((nextLine = reader.readNext()) != null) {
                if (firstLine.length != nextLine.length) {
                    throw new SmvcRuntimeException(PyramusStatusCode.VALIDATION_FAILURE, Messages.getInstance().getText(locale, "system.importcsv.invalidNumberOfArguments", new Object[] { rowNum }));
                }
                DataImportContext context = new DataImportContext(loggedUserId);
                context.setFields(firstLine);
                context.setFieldValues(nextLine);
                entityHandler.initializeContext(context);
                String value;
                String fieldName;
                for (int i = 0; i < firstLine.length; i++) {
                    fieldName = firstLine[i];
                    value = nextLine[i];
                    if (!StringUtils.isBlank(fieldName) && !StringUtils.isBlank(value))
                        entityHandler.handleValue(fieldName, value, context);
                }
                entityHandler.saveEntities(context);
                Object[] entities2 = context.getEntities();
                for (int i = 0; i < entities2.length; i++) {
                    if (entities2[i].getClass().equals(entityHandler.getMainEntityClass())) {
                        list.add(entities2[i]);
                    }
                }
                rowNum++;
            }
        }
    } catch (Exception e) {
        throw new SmvcRuntimeException(e);
    } finally {
        try {
            reader.close();
        } catch (IOException e) {
            throw new SmvcRuntimeException(e);
        }
    }
    return list;
}

public static List<String> getColumnIds(String path, char separator, boolean header) throws IOException {
    CSVReader reader = new CSVReader(new FileReader(path), separator);
    String[] headerLine = reader.readNext();
    reader.close();
    if (header) {
        return Arrays.asList(headerLine);
    } else {
        List<String> columnIDs = new ArrayList<String>(headerLine.length);
        for (int i = 0; i < headerLine.length; i++) {
            columnIDs.add(String.valueOf(i));
        }
        return columnIDs;
    }
}

@Test
public void testRunRegression() {
    CSVReader r = null;
    try {
        r = new CSVReader(new FileReader("src/test/resources/testRegression.tab"), '\t');
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    }
    IRudderList<MockRecord> list = new RudderList<MockRecord>();
    String[] line = null;
    try {
        while ((line = r.readNext()) != null) {
            MockRecord m = new MockRecord();
            try {
                m.setDoubleLabel(Double.parseDouble(line[0]));
                m.setFeature("feature1", Double.parseDouble(line[1]));
                m.setFeature("feature2", Double.parseDouble(line[2]));
                m.setFeature("feature3", Double.parseDouble(line[3]));
                list.add(m);
            } catch (NumberFormatException e) {
                e.printStackTrace();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    MultiLinearRegression ols = new MultiLinearRegression(list);
    double[] betas = ols.runRegression();
    assertEquals(12d, betas[0], .00001);
    assertEquals(4d, betas[1], .00001);
    assertEquals(6d, betas[2], .00001);
    assertEquals(5d, betas[3], .00001);
}

@Override
public void uploadFinished(Upload.FinishedEvent finishedEvent) {
    CSVReader csvReader = null;
    try {
        csvReader = new CSVReader(new InputStreamReader(new ByteArrayInputStream(csvData)));
        String[] record;
        while ((record = csvReader.readNext()) != null) {
            if (record.length != 3) {
                finishedEvent.getUpload().getWindow().showNotification("Должен быть csv файл Ñ? разделителем ','", Window.Notification.TYPE_ERROR_MESSAGE);
                return;
            }
            String dirName = record[0].trim();
            DirectoryBeanProvider.get().create(dirName);
            AdminServiceProvider.get().createLog(Flash.getActor(), "Directory", dirName, "create", "Upload create", true);
            String key = record[1].trim();
            String value = record[2].trim();
            DirectoryBeanProvider.get().add(dirName, key, value);
            AdminServiceProvider.get().createLog(Flash.getActor(), "Directory value", dirName, "add", "key => ".concat(key).concat("value =>").concat(value), true);
        }
    } catch (Exception e) {
        e.printStackTrace();
        return;
    } finally {
        if (csvReader != null) {
            try {
                csvReader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    ManagerWorkplace.refreshDirectoryTable(directoryTable);
    Object rowId = directoryTable.getValue();
    if (rowId != null) {
        final String dirName = (String) directoryTable.getContainerProperty(rowId, "name").getValue();
        ManagerWorkplace.reloadMap(dirName, dirMapTable);
    }
}

public String readFromCSV(String fileName) {
    File file = new File(fileName);
    FileInputStream fstream;
    DataInputStream in;
    BufferedReader br;
    CSVReader reader = null;
    StringBuffer buffer = new StringBuffer();
    try {
        fstream = new FileInputStream(file);
        in = new DataInputStream(fstream);
        br = new BufferedReader(new InputStreamReader(in));
        reader = new CSVReader(br, ',', '"', '\\');
        boolean first = true;
        String[] nextLine;
        String[] arrayHeaders = null;
        String[] arrayValues = null;
        while ((nextLine = reader.readNext()) != null) {
            String[] lineArr = nextLine;
            if (first) {
                arrayHeaders = lineArr;
                first = false;
            } else {
                arrayValues = lineArr;
                for (int i = 0; i < arrayValues.length; i++) {
                    buffer.append(arrayValues[i]);
                    buffer.append("\n");
                }
            }
        }
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (reader != null) {
            try {
                reader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    return buffer.toString();
}

/**
     * {@inheritDoc}
     */
@Override
public int read(InputStream in, ColumnInfo[] columns, Callback callback) {
    try {
        CSVReader reader = new CSVReader(new InputStreamReader(in, charset));
        int line = 0;
        int count = 0;
        String[] values = null;
        while ((values = reader.readNext()) != null) {
            line++;
            if (values.length == 0) {
                continue;
            }
            if (columns.length != values.length) {
                Log.w("Fixtures", String.format("ignore line: %d", line));
                continue;
            }
            Row row = new Row(++count);
            for (int i = 0; i < values.length; i++) {
                row.add(columns[i], values[i]);
            }
            callback.handleRow(row);
        }
        return count;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    try {
        reader = new CSVReader(new FileReader(fileName), separator);
        // read and ignore the header if one exists
        if (includesHeaderRow)
            reader.readNext();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

@SuppressWarnings("unchecked")
@Override
public void onAddToLayout(String portletId, long plid) throws PortletLayoutListenerException {
    try {
        Layout layout = LayoutLocalServiceUtil.getLayout(plid);
        long companyId = layout.getCompanyId();
        long groupId = layout.getGroupId();
        ColumnPositionMappingStrategy strat = new ColumnPositionMappingStrategy();
        strat.setType(InsultImpl.class);
        String[] columns = new String[] { "adjectiveOne", "directObject", "adjectiveTwo", "objPreposition" };
        strat.setColumnMapping(columns);
        CsvToBean csv = new CsvToBean();
        ClassLoader classLoader = this.getClass().getClassLoader();
        CSVReader reader = new CSVReader(new InputStreamReader(classLoader.getResourceAsStream("insults.csv")));
        List list = csv.parse(strat, reader);
        Insult testInsult = (Insult) list.get(0);
        if (InsultLocalServiceUtil.getInsultByObjPreposition(testInsult.getObjPreposition()) == null) {
            for (int i = 0; i < list.size(); i++) {
                Insult insult = (Insult) list.get(i);
                insult.setCompanyId(companyId);
                insult.setGroupId(groupId);
                InsultLocalServiceUtil.addInsult(insult, 0);
            }
        }
    } catch (PortalException e) {
        System.out.println("PortalException: " + e.getMessage());
    } catch (SystemException e) {
        System.out.println("SystemException: " + e.getMessage());
    }
}

/**
	 * csv file reader
	 * 
	 * @param filename
	 * @param seprator
	 * @return
	 * @throws Exception
	 */
public static List<String[]> readFile(String filename, char seprator) {
    List<String[]> returnMap = new ArrayList<String[]>();
    String[] nextLine = null;
    CSVReader csvReader = null;
    try {
        csvReader = new CSVReader(new BufferedReader(new InputStreamReader(new FileInputStream(filename), "MS949")), seprator);
        while ((nextLine = csvReader.readNext()) != null) {
            returnMap.add(nextLine);
        }
    } catch (Exception e) {
        logger.error("CSV read error", e);
    } finally {
        if (csvReader != null)
            try {
                csvReader.close();
            } catch (Exception e) {
            }
    }
    return returnMap;
}

@DataProvider
public static Object[][] csvDataProvider(Method m) throws IOException {
    if (m.isAnnotationPresent(CsvDataSource.class)) {
        int length = m.getParameterTypes().length;
        CsvDataSource dataSource = m.getAnnotation(CsvDataSource.class);
        File csvFile = new File(dataSource.value());
        List<Object[]> result = new ArrayList<Object[]>();
        CSVReader reader = new CSVReader(new FileReader(csvFile));
        String[] nextLine;
        while ((nextLine = reader.readNext()) != null) {
            System.out.println(Arrays.toString(nextLine));
            Object[] parameters = new Object[length];
            for (int i = 0; i < length; i++) {
                if (i < nextLine.length) {
                    parameters[i] = nextLine[i];
                } else {
                    parameters[i] = null;
                }
            }
            result.add(parameters);
        }
        return result.toArray(new Object[result.size()][]);
    } else {
        throw new Error("There is no @CsvDataSource annotation on method " + m);
    }
}

@Override
public void prepare(Map conf, TridentOperationContext context) {
    if (conf.get(backtype.storm.Config.TOPOLOGY_DEBUG).equals(true)) {
        testMode = true;
        CSVReader reader;
        try {
            reader = new CSVReader(new BufferedReader(new InputStreamReader(DocumentFetchFunction.class.getResourceAsStream("docs.csv"))));
            List<String[]> myEntries = reader.readAll();
            for (String[] row : myEntries) {
                testData.put(row[0], row[1]);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

// todo serso: due to conversion errors values on the borders are calculated not precisely
/*
 0;0;1;0;Infinity
 90;1;0;1.633123935319537E16;0
 180;0;-1;0;-8.165619676597685E15
 270;-1;0;5.443746451065123E15;0
 360;0;1;0;-4.0828098382988425E15
      */
@Test
public void testValues() throws Exception {
    CSVReader reader = null;
    try {
        final MathEngine me = JsclMathEngine.getInstance();
        reader = new CSVReader(new InputStreamReader(TrigonometricTest.class.getResourceAsStream("./trig_table.csv")), '\t');
        // skip first line
        reader.readNext();
        String[] line = reader.readNext();
        for (; line != null; line = reader.readNext()) {
            final Integer degrees = Integer.valueOf(line[0]);
            final Double sinValue = Double.valueOf(line[1]);
            final Double cosValue = Double.valueOf(line[2]);
            final Double tgValue = Double.valueOf(line[3]);
            final Double ctgValue = Double.valueOf(line[4]);
            final Double radians = Double.valueOf(line[5]);
            final Double sinhValue = Double.valueOf(line[6]);
            final Double coshValue = Double.valueOf(line[7]);
            final Double tghValue = Double.valueOf(line[8]);
            final Double cthgValue = Double.valueOf(line[9]);
            final Double asinValue = Double.valueOf(line[10]);
            final Double acosValue = Double.valueOf(line[11]);
            final Double atanValue = Double.valueOf(line[12]);
            testValue(sinValue, Double.valueOf(me.evaluate("sin(" + degrees + "°)")), degrees);
            testValue(cosValue, Double.valueOf(me.evaluate("cos(" + degrees + "°)")), degrees);
            testValue(tgValue, Double.valueOf(me.evaluate("tan(" + degrees + "°)")), degrees);
            testValue(ctgValue, Double.valueOf(me.evaluate("cot(" + degrees + "°)")), degrees);
            testValue(sinhValue, Double.valueOf(me.evaluate("sinh(" + degrees + "°)")), degrees);
            testValue(coshValue, Double.valueOf(me.evaluate("cosh(" + degrees + "°)")), degrees);
            testValue(tghValue, Double.valueOf(me.evaluate("tanh(" + degrees + "°)")), degrees);
            testValue(cthgValue, Double.valueOf(me.evaluate("coth(" + degrees + "°)")), degrees);
            final AngleUnit angleUnits = me.getAngleUnits();
            try {
                me.setAngleUnits(AngleUnit.rad);
                testValue(sinValue, Double.valueOf(me.evaluate("sin(" + radians + ")")), degrees);
                testValue(cosValue, Double.valueOf(me.evaluate("cos(" + radians + ")")), degrees);
                testValue(tgValue, Double.valueOf(me.evaluate("tan(" + radians + ")")), degrees);
                testValue(ctgValue, Double.valueOf(me.evaluate("cot(" + radians + ")")), degrees);
                testValue(sinhValue, Double.valueOf(me.evaluate("sinh(" + radians + ")")), degrees);
                testValue(coshValue, Double.valueOf(me.evaluate("cosh(" + radians + ")")), degrees);
                testValue(tghValue, Double.valueOf(me.evaluate("tanh(" + radians + ")")), degrees);
                testValue(cthgValue, Double.valueOf(me.evaluate("coth(" + radians + ")")), degrees);
            } finally {
                me.setAngleUnits(angleUnits);
            }
            testValue(asinValue, Double.valueOf(me.evaluate("rad(asin(" + sinValue + "))")), degrees);
            testValue(acosValue, Double.valueOf(me.evaluate("rad(acos(" + cosValue + "))")), degrees);
            testValue(atanValue, Double.valueOf(me.evaluate("rad(atan(" + tgValue + "))")), degrees);
            // todo serso: check this
            //testValue((double)degrees, Double.valueOf(me.evaluate("asin(sin(" + degrees + "°))")), degrees);
            //testValue((double)degrees, Double.valueOf(me.evaluate("acos(cos(" + degrees + "°))")), degrees);
            //testValue((double)degrees, Double.valueOf(me.evaluate("atan(tan(" + degrees + "°))")), degrees);
            //testValue((double)degrees, Double.valueOf(me.evaluate("acot(cot(" + degrees + "°))")), degrees);
            testValue(sinValue, Double.valueOf(me.evaluate("sin(asin(sin(" + degrees + "°)))")), degrees);
            testValue(cosValue, Double.valueOf(me.evaluate("cos(acos(cos(" + degrees + "°)))")), degrees);
            testValue(tgValue, Double.valueOf(me.evaluate("tan(atan(tan(" + degrees + "°)))")), degrees);
            testValue(ctgValue, Double.valueOf(me.evaluate("cot(acot(cot(" + degrees + "°)))")), degrees);
        }
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
}

public void buildSkillsLibrary() throws IOException, FileNotFoundException {
    say("Searching for file in: " + new File(".").getAbsolutePath());
    CSVReader reader = new CSVReader(new FileReader(filename));
    String[] nextLine;
    while ((nextLine = reader.readNext()) != null) {
        Skill skill = new Skill(nextLine[0], nextLine[1], skills.size() + 1);
        skills.add(skill);
        say("Skill " + skill.getId() + ": " + skill.getName() + " added to factory");
    }
    reader.close();
}

@Before
public void setUp() throws Exception {
    File file = File.createTempFile("cp1", "file");
    FileWriter writer = new FileWriter(file);
    IOUtils.write(FILE_CONTENT, writer);
    IOUtils.closeQuietly(writer);
    configuration = mock(ISingleFileConfiguration.class);
    when(configuration.getFile()).thenReturn(file);
    parser = spy(new CSVParser());
    dataIndex = 0;
    reader = mock(CSVReader.class);
    when(reader.readNext()).thenAnswer(new CSV_READER_ANSWER());
}

private static final WorldCity[] populateCities(Context context, int skipInterval) {
    List<String[]> citiesList = new ArrayList<String[]>();
    AssetManager assetManager = context.getAssets();
    try {
        InputStream csvStream = assetManager.open("cities.csv");
        InputStreamReader csvStreamReader = new InputStreamReader(csvStream);
        CSVReader csvReader = new CSVReader(csvStreamReader);
        String[] line;
        // throw away the header
        csvReader.readNext();
        int skipCounter = 0;
        while ((line = csvReader.readNext()) != null) if (skipCounter == skipInterval) {
            citiesList.add(line);
            skipCounter = 0;
        } else
            skipCounter++;
        CITIES = new WorldCity[citiesList.size()];
        for (int i = 0; i < citiesList.size(); i++) SeedDataUtil.CITIES[i] = new WorldCity(ID_SENTINAL++, citiesList.get(i)[0]);
        csvReader.close();
    } catch (IOException e) {
        LAST_SKIP_INTERVAL = skipInterval;
        e.printStackTrace();
    }
    LAST_SKIP_INTERVAL = skipInterval;
    return CITIES;
}

public static void main(String[] args) throws Exception {
    File file = new File(args[0]);
    List<String[]> input;
    try (FileInputStream fis = new FileInputStream(file);
        InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
        CSVReader csvr = new CSVReader(isr, ';', CSVWriter.DEFAULT_QUOTE_CHARACTER)) {
        input = csvr.readAll();
    }
    Configuration configuration = new Configuration();
    BrickLinkClient client = new BrickLinkClient(configuration);
    List<String[]> output = new ArrayList<String[]>();
    try {
        for (String[] line : input) {
            List<String> row = new ArrayList<String>();
            for (String tmp : line) {
                row.add(tmp);
            }
            String legoId = line[0];
            int identifier;
            try {
                identifier = Integer.parseInt(legoId);
            } catch (NumberFormatException e) {
                throw e;
            }
            ItemNumberResponse response = client.execute(new ItemNumberRequest(identifier));
            if (response.getItemMapping().size() != 1) {
                System.out.println(response.getItemMapping().toString());
                output.add(row.toArray(new String[row.size()]));
                continue;
            }
            ItemMapping mapping = response.getItemMapping().get(0);
            System.out.println(mapping.toString());
            String itemID = mapping.getItem().getIdentifier();
            row.add(itemID);
            int colorID = mapping.getColor().getIdentifier();
            row.add(Integer.toString(colorID));
            ItemType type = mapping.getType() == null ? mapping.getItem().getType() : mapping.getType();
            PriceGuide soldGuide = client.execute(new PriceGuideRequest(type, itemID, colorID, GuideType.SOLD, Condition.N)).getPriceGuide();
            PriceGuide stockGuide = client.execute(new PriceGuideRequest(type, itemID, colorID, GuideType.STOCK, Condition.N)).getPriceGuide();
            BigDecimal quantityAveragePrice = soldGuide.getQuantityAveragePrice();
            row.add(quantityAveragePrice.toString());
            row.add(Integer.toString(stockGuide.getQuantity()));
            row.add(Integer.toString(soldGuide.getQuantity()));
            output.add(row.toArray(new String[row.size()]));
        }
    } finally {
        client.close();
    }
    FileOutputStream fis = null;
    OutputStreamWriter osw = null;
    CSVWriter csvw = null;
    try {
        fis = new FileOutputStream(new File("lugbulk.csv"));
        osw = new OutputStreamWriter(fis, "UTF-8");
        csvw = new CSVWriter(osw, ';', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER, "\n");
        csvw.writeAll(output);
    } finally {
        IOUtils.closeQuietly(osw);
        IOUtils.closeQuietly(fis);
        if (null != csvw) {
            csvw.close();
        }
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

// Return rows added
private int addSamples(ControlFile controlFile) throws IOException {
    CSVReader reader = getCSVReader(controlFile, controlFile.getFileTypeControl().skip);
    String[] row = reader.readNext();
    int rowsAdded = 0;
    while (row != null && rowsAdded < rowsToSample) {
        // control file editor the ability to load.
        if (isBlankRow(row)) {
            insertData(getBlankPlaceholderRow(getColumnCount()));
        } else {
            insertData(row);
        }
        rowsAdded++;
        row = reader.readNext();
    }
    return rowsAdded;
}

private void importCSV(ImportData importData, ImportDriverInfo driver, Reader reader) throws IOException {
    final CSVReader csvReader = new CSVReader(reader);
    final String[] header = csvReader.readNext();
    if (ArrayUtils.isEmpty(header)) {
        setErrorMessage("No connection found");
        return;
    }
    for (; ; ) {
        final String[] line = csvReader.readNext();
        if (ArrayUtils.isEmpty(line)) {
            break;
        }
        Map<String, String> conProps = new HashMap<>();
        for (int i = 0; i < line.length; i++) {
            if (i >= header.length) {
                break;
            }
            conProps.put(header[i], line[i]);
        }
        makeConnectionFromProps(importData, driver, conProps);
    }
}

public List<RuleOp> load(File file) throws FileNotFoundException, IOException {
    CSVReader reader = new CSVReader(new FileReader(file), config.getFieldSeparator().charAt(0));
    // fetch layer names at given columns
    String[] headers = reader.readNext();
    for (RuleFileConfig.Group group : config.getGroups()) {
        // indices are 1-based
        int idx = group.getIndex() - 1;
        if (headers.length <= idx)
            throw new IndexOutOfBoundsException("Header has " + headers.length + " elements. " + "Group column expected at position " + (idx + 1));
        String realName = headers[idx];
        LOGGER.debug(" Group name in rule file : '" + realName + "'");
    //            groupNames.add(realName);
    }
    final Pattern namePattern = config.getValidLayernameRegEx() != null ? Pattern.compile(config.getValidLayernameRegEx()) : null;
    List<RuleOp> ret = new LinkedList<RuleOp>();
    String[] line;
    while ((line = reader.readNext()) != null) {
        if (line.length < config.getLayerNameIndex())
            throw new IndexOutOfBoundsException("Line has " + line.length + " elements. " + "Layer expected at position " + config.getLayerNameIndex());
        String layername = line[config.getLayerNameIndex() - 1];
        if (namePattern != null && !namePattern.matcher(layername).matches()) {
            LOGGER.debug("Discarding layer '" + layername + "' by regexp");
            continue;
        }
        for (RuleFileConfig.Group group : config.getGroups()) {
            // indices are 1-based in config
            int idx = group.getIndex() - 1;
            String groupName = headers[idx];
            if (line.length <= idx)
                throw new IndexOutOfBoundsException("Line has " + line.length + " elements. " + "Rule for group '" + groupName + "' expected at position " + (idx + 1));
            String verb = line[idx];
            //                List<RuleFileConfig.ServiceRequest> configRules = config.getRuleMapping().get(verb);
            //                if(configRules == null) {
            //                    LOGGER.warn("Unknown verb '"+verb+"' for layer '" + layername + "' and group '"+groupName+"'");
            //                    continue;
            //                }
            //                if(! configRules.isEmpty()) {
            final RuleOp ruleOp = new RuleOp();
            ruleOp.setLayerName(layername);
            ruleOp.setGroupName(groupName);
            ruleOp.setVerb(verb);
            ret.add(ruleOp);
        //                }
        }
    }
    return ret;
}

/**
	 * @param args
	 * @throws IOException 
	 */
public static void main(String[] args) throws IOException {
    log.info("Reading matches.csv.");
    File directory = new File("/home/martin/Desktop/tmp");
    File matchesFile = new File(directory, "rawdata.csv");
    CSVReader reader = new CSVReader(new FileReader(matchesFile));
    List<String[]> lines = new LinkedList<String[]>();
    for (Iterator<?> iter = reader.readAll().iterator(); iter.hasNext(); ) {
        String[] line = (String[]) iter.next();
        lines.add(line);
    }
    for (String[] line : lines) {
        // "Matchid" , "Player1;Player2;...", "Game", "Reward1;Reward2;..."
        if (line.length != 4) {
            throw new IllegalArgumentException("Incorrect number of arguments in CSV file " + matchesFile);
        }
        String matchID = line[0];
        String players = line[1];
        //	    	String game    = line[2];
        String rewards = line[3];
        StringTokenizer playerTokens = new StringTokenizer(players, ";");
        StringTokenizer rewardTokens = new StringTokenizer(rewards, ";");
        int numPlayers = playerTokens.countTokens();
        assert (rewardTokens.countTokens() == numPlayers);
        //	    	Writer output = new PrintWriter(System.out);
        Writer output = new FileWriter(new File(directory, matchID + ".xml"));
        output.write("<?xml version=\"1.0\"?>\n");
        output.write("<!DOCTYPE match SYSTEM \"http://games.stanford.edu/gamemaster/xml/viewmatch.dtd\">\n");
        output.write("<match>\n");
        // --- match id ---
        output.write("\t<match-id>" + matchID + "</match-id>\n");
        // --- roles ---
        for (int i = 1; i <= numPlayers; i++) {
            output.write("\t<role>role" + i + "</role>\n");
        }
        // --- players ---
        while (playerTokens.hasMoreTokens()) {
            output.write("\t<player>" + playerTokens.nextToken() + "</player>\n");
        }
        // --- scores ---
        output.write("\t<scores>\n");
        while (rewardTokens.hasMoreTokens()) {
            output.write("\t\t<reward>" + rewardTokens.nextToken() + "</reward>\n");
        }
        output.write("\t</scores>\n");
        output.write("</match>\n");
        // flush instead if PrintWriter 
        output.close();
    }
}

static void readData(File f, int[] mapping, String firstColName, double[][] out, MojoModel mojo) throws IOException {
    InputStream is = new FileInputStream(f);
    try {
        InputStream source;
        if (f.getName().endsWith(".zip")) {
            ZipInputStream zis = new ZipInputStream(is);
            ZipEntry entry = zis.getNextEntry();
            if (!entry.getName().endsWith(".csv"))
                throw new IllegalStateException("CSV file expected, name " + entry.getName());
            source = zis;
        } else {
            source = new GZIPInputStream(is);
        }
        CSVReader r = new CSVReader(new InputStreamReader(source));
        if (firstColName != null) {
            String[] header = r.readNext();
            if (header == null)
                throw new IllegalStateException("File empty");
            if (!firstColName.equals(header[0]))
                throw new IllegalStateException("Header expected");
        }
        int rowIdx = 0;
        String[] row;
        while ((rowIdx < out.length) && ((row = r.readNext()) != null)) {
            double[] outRow = out[rowIdx++];
            if (row.length < mapping.length)
                throw new IllegalStateException("Row too short: " + Arrays.toString(row));
            for (int i = 0; i < mapping.length; i++) {
                int target = mapping[i];
                if (target < 0)
                    continue;
                if ("NA".equals(row[i])) {
                    outRow[target] = Double.NaN;
                    continue;
                }
                String[] domain = mojo.getDomainValues(target);
                if (domain == null)
                    outRow[target] = Double.parseDouble(row[i]);
                else {
                    outRow[target] = -1;
                    for (int d = 0; d < domain.length; d++) if (domain[d].equals(row[i])) {
                        outRow[target] = d;
                        break;
                    }
                    if (outRow[target] < 0)
                        throw new IllegalStateException("Value " + row[i] + " not found in domain " + Arrays.toString(domain));
                }
            }
        }
    } finally {
        is.close();
    }
}

/**
     * @see org.kuali.kfs.module.tem.batch.service.PerDiemFileParsingService#buildPerDiemsFromFlatFile(java.io.Reader,
     *      java.lang.String)
     */
@Override
public List<PerDiemForLoad> buildPerDiemsFromFlatFile(Reader reader, String deliminator, List<String> fieldsToPopulate) {
    List<PerDiemForLoad> perDiemList = new ArrayList<PerDiemForLoad>();
    CSVReader csvReader = null;
    try {
        char charDeliminator = deliminator.charAt(0);
        csvReader = new CSVReader(reader, charDeliminator);
        String[] perDiemInString = null;
        while ((perDiemInString = csvReader.readNext()) != null) {
            if (ArrayUtils.contains(perDiemInString, "FOOTNOTES: ")) {
                break;
            }
            PerDiemForLoad perDiem = new PerDiemForLoad();
            ObjectUtil.buildObject(perDiem, perDiemInString, fieldsToPopulate);
            perDiemList.add(perDiem);
        }
    } catch (Exception ex) {
        LOG.error("Failed to process data file. ");
        throw new RuntimeException("Failed to process data file. ", ex);
    } finally {
        if (csvReader != null) {
            try {
                csvReader.close();
            } catch (IOException ex) {
                LOG.info(ex);
            }
        }
        IOUtils.closeQuietly(reader);
    }
    return perDiemList;
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

private Map<String, List<Entity>> getEntitiesByGeneSymbol() {
    if (entitiesByGeneSymbol == null) {
        entitiesByGeneSymbol = new LinkedHashMap<>();
        try (CSVReader csvReader = new CSVReader(new InputStreamReader(new FileInputStream(file), Charset.forName("UTF-8")), '\t', CSVParser.DEFAULT_QUOTE_CHARACTER, 1)) {
            String[] values = csvReader.readNext();
            while (values != null) {
                String geneSymbol = values[1];
                Entity entity = new DynamicEntity(getEntityType());
                entity.set(HPO_DISEASE_ID_COL_NAME, values[0]);
                entity.set(HPO_GENE_SYMBOL_COL_NAME, geneSymbol);
                entity.set(HPO_ID_COL_NAME, values[3]);
                entity.set(HPO_TERM_COL_NAME, values[4]);
                List<Entity> entities = entitiesByGeneSymbol.get(geneSymbol);
                if (entities == null) {
                    entities = new ArrayList<>();
                    entitiesByGeneSymbol.put(geneSymbol, entities);
                }
                entities.add(entity);
                values = csvReader.readNext();
            }
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
    return entitiesByGeneSymbol;
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

private static final WorldCity[] populateCities(Context context, int skipInterval) {
    List<String[]> citiesList = new ArrayList<String[]>();
    AssetManager assetManager = context.getAssets();
    try {
        InputStream csvStream = assetManager.open("cities.csv");
        InputStreamReader csvStreamReader = new InputStreamReader(csvStream);
        CSVReader csvReader = new CSVReader(csvStreamReader);
        String[] line;
        // throw away the header
        csvReader.readNext();
        int skipCounter = 0;
        while ((line = csvReader.readNext()) != null) if (skipCounter == skipInterval) {
            citiesList.add(line);
            skipCounter = 0;
        } else
            skipCounter++;
        CITIES = new WorldCity[citiesList.size()];
        for (int i = 0; i < citiesList.size(); i++) SeedDataUtil.CITIES[i] = new WorldCity(ID_SENTINAL++, citiesList.get(i)[0]);
        csvReader.close();
    } catch (IOException e) {
        LAST_SKIP_INTERVAL = skipInterval;
        e.printStackTrace();
    }
    LAST_SKIP_INTERVAL = skipInterval;
    return CITIES;
}

@Override
protected void doRun() {
    try {
        String data = get(new URL(url.replaceAll("CURRENCY", base.getId())));
        List<String[]> entries = new ArrayList<String[]>();
        StringReader strReader = new StringReader(data);
        CSVReader csvReader = new CSVReader(strReader);
        try {
            entries = csvReader.readAll();
        } finally {
            strReader.close();
            csvReader.close();
        }
        if (entries == null || entries.size() == 0)
            throw new CSVStructureException();
        /* Ensure that the headers match */
        if (headers.length != entries.get(0).length) {
            logger.error("Expected length " + headers.length + " got " + entries.get(0).length);
            throw new CSVStructureException();
        }
        for (int i = 0; i < headers.length; i++) {
            if (!headers[i].equals(entries.get(0)[i]))
                throw new CSVStructureException();
        }
        entries.remove(0);
        rates.clear();
        for (String[] fields : entries) {
            long timestamp = parseDate(fields[DATETIME_INDEX]);
            double avarage = Double.parseDouble(fields[AVERAGE_INDEX]);
            IRate rate = new Rate(base, Currencies.BTC, timestamp, avarage, 0);
            rates.add(rate);
            for (Object sink : sinks) {
                ((IRateSink) sink).addRate(rate);
            }
        }
        notifyListeners();
    } catch (MalformedURLException e) {
        logger.error("Mallformed URL " + url, e);
    } catch (HTTPDataProviderException e) {
        logger.error("HTTPDataProviderException " + url, e);
    } catch (IOException e) {
        logger.error("IOException " + url, e);
    } catch (CSVStructureException e) {
        logger.error("IOException " + url, e);
    }
}

@Override
@Test(enabled = false)
public void init() throws Exception {
    String buildDirectory = System.getProperty("buildDirectory", ".");
    if (buildDirectory == null)
        buildDirectory = ".";
    databaseDocumentTx = new ODatabaseDocumentTx("plocal:" + buildDirectory + "/location");
    if (databaseDocumentTx.exists()) {
        databaseDocumentTx.open("admin", "admin");
        databaseDocumentTx.drop();
    }
    databaseDocumentTx.create();
    OSchema schema = databaseDocumentTx.getMetadata().getSchema();
    OClass oClass = schema.createClass("City");
    oClass.createProperty("latitude", OType.DOUBLE);
    oClass.createProperty("longitude", OType.DOUBLE);
    oClass.createProperty("name", OType.STRING);
    oClass.createIndex("City.latitude_longitude", "SPATIAL", null, null, "LUCENE", new String[] { "latitude", "longitude" });
    oClass.createIndex("City.name", "FULLTEXT", null, null, "LUCENE", new String[] { "name" });
    ZipFile zipFile = new ZipFile("files/location.csv.zip");
    Enumeration<? extends ZipEntry> entries = zipFile.entries();
    while (entries.hasMoreElements()) {
        ZipEntry entry = entries.nextElement();
        if (entry.getName().equals("location.csv")) {
            InputStream stream = zipFile.getInputStream(entry);
            reader = new CSVReader(new InputStreamReader(stream), ',');
        }
    }
}

public List<T> parse(MappingStrategy<T> mapper, CSVReader csv) {
    try {
        mapper.captureHeader(csv);
        String[] line;
        List<T> list = new ArrayList<T>();
        while (null != (line = csv.readNext())) {
            T obj = processLine(mapper, line);
            // TODO: (Kyle) null check object
            list.add(obj);
        }
        return list;
    } catch (Exception e) {
        throw new RuntimeException("Error parsing CSV!", e);
    }
}

private int getNumColumns(FilePath workspaceRootDir, String file) throws IOException, InterruptedException {
    CSVReader csvreader = null;
    InputStream in = null;
    InputStreamReader inputReader = null;
    FilePath[] seriesFiles = null;
    try {
        seriesFiles = workspaceRootDir.list(file);
        if (seriesFiles != null && seriesFiles.length < 1) {
            LOGGER.info("No plot data file found: " + workspaceRootDir.getName() + " " + file);
            return -1;
        }
        LOGGER.info("Loading plot series data from: " + file);
        in = seriesFiles[0].read();
        inputReader = new InputStreamReader(in);
        csvreader = new CSVReader(inputReader);
        // save the header line to use it for the plot labels.
        String[] headerLine = csvreader.readNext();
        LOGGER.info("Got " + headerLine.length + " columns");
        return headerLine.length;
    } finally {
        try {
            if (csvreader != null)
                csvreader.close();
        } catch (IOException e) {
        }
        IOUtils.closeQuietly(inputReader);
        IOUtils.closeQuietly(in);
    }
}

@Override
public synchronized void loadRawData() throws Exception {
    //Temporary variables.
    ArrayList<MarketGroup> tmpDb = new ArrayList<>();
    ArrayList<String[]> allGroups = new ArrayList<>();
    ArrayList<Integer> tmpGroups = new ArrayList<>();
    String[] nextLine;
    //Parse Top level MarketGroups.
    CSVReader csv = new CSVReader(new FileReader(MARKET_GROUP_PATH), ';');
    //Skip header.
    csv.readNext();
    //Add all topLevel parents to the list.
    while ((nextLine = csv.readNext()) != null) {
        //If group has no parent.
        if (nextLine[1].equals("")) {
            tmpDb.add(new MarketGroup(nextLine[2], Integer.parseInt(nextLine[0]), nextLine[3]));
        }
    }
    csv.close();
    //Read all.
    csv = new CSVReader(new FileReader(MARKET_GROUP_PATH), ';');
    //Skip header.
    csv.readNext();
    allGroups.addAll(csv.readAll());
    csv.close();
    for (MarketGroup mg : tmpDb) {
        tmpGroups.clear();
        tmpGroups.add(mg.getGroupId());
        while (!hasOnlyItemGroups(tmpGroups, allGroups)) {
            tmpGroups = getChildGroups(tmpGroups, allGroups);
        }
        mg.getChildGroups().addAll(tmpGroups);
    }
    //Set new database to global reference.
    db = tmpDb;
    //Show message.
    EMT.M_HANDLER.addMessage("Marketgroups and metadata loaded.");
    //Last initiation step, set complete.
    super.setComplete(true);
}

public void convertToHtml(File csvFile, File htmlFile, boolean withTableHeader) throws IOException {
    try (CSVReader csvReader = new CSVReader(new FileReader(csvFile));
        BufferedWriter htmlWriter = new BufferedWriter(new FileWriter(htmlFile))) {
        String lineOfCsvValues[];
        boolean firstLine = true;
        htmlBodyTableOpening(htmlWriter);
        while ((lineOfCsvValues = csvReader.readNext()) != null) {
            if (isBlankLine(lineOfCsvValues))
                continue;
            String[] columns = escapeChars(lineOfCsvValues);
            if (withTableHeader == true && firstLine == true) {
                tableHeader(htmlWriter, columns);
                firstLine = false;
                tableBodyOpening(htmlWriter);
            } else {
                tableRow(htmlWriter, columns);
            }
        }
        tableBodyClosing(htmlWriter);
        htmlBodyTableClosing(htmlWriter);
        htmlWriter.close();
    }
}

/**
     * Reads lesson files data.
     * @param fileName Name of the file
     * @param onlyMeta If true only file header with lesson meta info will be read
     * @return Lesson object
     * @throws DataException
     */
private Lesson readUserLesson(String fileName, boolean onlyMeta) throws DataException {
    Lesson lesson = new Lesson();
    ArrayList<Card> cards = new ArrayList<Card>();
    try {
        // open file
        FileInputStream fileInputStream = context.openFileInput(fileName);
        InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream, "UTF-8");
        CSVReader reader = new CSVReader(inputStreamReader, ';', '\"', 0);
        String[] nextLine;
        if ((nextLine = reader.readNext()) != null) {
            // read meta info from the first line
            lesson = new Lesson(nextLine[0], nextLine[1], nextLine[2], nextLine[3], nextLine[4], Integer.parseInt(nextLine[5]));
            lesson.setId(DataHelper.md5(lesson.getFilename() + lesson.getCardCount()));
            lesson.setSourceType(SourceType.INTERNAL);
            if (!onlyMeta) {
                while ((nextLine = reader.readNext()) != null) {
                    String faceText = nextLine[0].trim();
                    String backText = nextLine[1].trim();
                    Card c = new Card(DataHelper.md5(faceText), faceText, backText, false);
                    cards.add(c);
                }
                lesson.setCards(cards);
            } else {
                lesson.setCards(new ArrayList<Card>());
            }
        }
        reader.close();
        fileInputStream.close();
    } catch (UnsupportedEncodingException ex) {
        Log.e(TAG, ex.getMessage());
        throw new DataException(ERROR_CANNOT_READ_LESSON_FILE, ex);
    } catch (Exception e) {
        Log.e(TAG, e.getMessage());
        throw new DataException(ERROR_CANNOT_READ_LESSON_FILE, e);
    }
    return lesson;
}

@VisibleForTesting
static List<FileLinkerLine> getFileInfo(Reader reader, char separator) {
    CSVReader csvReader = new CSVReader(reader, separator);
    HeaderColumnNameTranslateMappingStrategy<FileLinkerLine> strat = new HeaderColumnNameTranslateMappingStrategy<>();
    strat.setType(FileLinkerLine.class);
    Map<String, String> map = Maps.newHashMap();
    map.put("sequencer_run", "sequencerRun");
    map.put("sample", "sample");
    map.put("lane", "laneString");
    map.put("ius_sw_accession", "seqwareAccessionString");
    map.put("file_status", "fileStatus");
    map.put("mime_type", "mimeType");
    map.put("size", "sizeString");
    map.put("md5sum", "md5sum");
    map.put("file", "filename");
    strat.setColumnMapping(map);
    CsvToBean<FileLinkerLine> csvToBean = new CsvToBean<>();
    List<FileLinkerLine> defaultUsers = csvToBean.parse(strat, csvReader);
    return defaultUsers;
}

private <T> List<T> load(Class<T> beanClass) {
    ColumnPositionMappingStrategy<T> strat = new ColumnPositionMappingStrategy<T>();
    strat.setType(beanClass);
    if (columnMapping == null) {
        throw new RuntimeException("columnMapping must to be set!");
    }
    strat.setColumnMapping(columnMapping);
    if (csvFile == null && csvSrc == null) {
        throw new RuntimeException("csvFile or csvSrc must to be set!");
    }
    if (skipLines < 0) {
        throw new RuntimeException("skipLines < 0 !");
    }
    InputStream is = null;
    try {
        is = csvSrc != null ? csvSrc : new FileInputStream(csvFile);
        CSVReader reader = new CSVReader(new InputStreamReader(is, charset), CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, skipLines);
        CsvToBean<T> csv = new CsvToBean<T>();
        return csv.parse(strat, reader);
    } catch (FileNotFoundException e) {
        throw new RuntimeException("CSV File not found!", e);
    } finally {
        closeInputStream(is);
    }
}

private void transform(File inputFile, List<Integer> columnOrder) throws IOException {
    CSVReader reader = new CSVReader(new FileReader(inputFile));
    String[] nextLine;
    while ((nextLine = reader.readNext()) != null) {
        int index = 0;
        for (int value : columnOrder) {
            System.out.print(nextLine[value].trim());
            index++;
            if (index < columnOrder.size()) {
                System.out.print(",");
            }
        }
        System.out.println();
    }
    reader.close();
}

public void testExportOneQuery() throws IOException {
    QueryLogStatistic stat = mock(QueryLogStatistic.class);
    LinkedList<QueryLogValue> values = new LinkedList<QueryLogValue>();
    QueryLogValue value = newMockQueryLogValue(false, "queryString", "facetQueryString", "filterQueryString", 123, 456l);
    values.add(value);
    when(stat.getLastQueries()).thenReturn(values);
    FullQueryStatisticController controller = new FullQueryStatisticController(stat);
    controller.writeCSV(temp);
    CSVReader reader = new CSVReader(new FileReader(temp));
    assertEquals(getStringsArrayFor(value), reader.readNext());
    assertNull(reader.readNext());
}

/**
     * Loads a CSV into a map keyed by enum (0-based column 
     * index is the enum ordinal)
     * 
     * @param <T>
     * @param enumClass
     * @param stream
     * @param skipFirstLine Whether or not to skip the first line
     * @return
     * @throws IOException
     */
@SuppressWarnings("unchecked")
public static <T extends Enum<T>> List<Map<T, String>> readCsv(Class<T> enumClass, Reader reader, boolean skipFirstLine) throws IOException {
    T[] enumValues;
    try {
        enumValues = (T[]) enumClass.getMethod("values").invoke(null);
    } catch (Exception e) {
        throw new BaseRuntimeException(e);
    }
    CSVReader csvReader = new CSVReader(reader);
    List<Map<T, String>> ret = new ArrayList<Map<T, String>>();
    String[] line;
    if (skipFirstLine) {
        csvReader.readNext();
    }
    while ((line = csvReader.readNext()) != null) {
        assert enumValues.length >= line.length : "Value count [" + line.length + "] greater than max [" + enumValues.length + "]";
        try {
            Map<T, String> map = new HashMap<T, String>(line.length);
            for (int i = 0; i < line.length; i++) {
                map.put(enumValues[i], line[i]);
            }
            ret.add(map);
        } catch (Exception e) {
            if (logger.isWarnEnabled()) {
                logger.warn("Unable to read line: " + Arrays.toString(line), e);
            }
        }
    }
    return ret;
}

public static void main(String[] args) {
    File dir = new File("/Users/bowu/Research/dataclean/data/RuleData");
    File[] flist = dir.listFiles();
    try {
        //BufferedWriter bw = new BufferedWriter(new FileWriter("/Users/bowu/Research/dataclean/data/negadata.out"));
        boolean isfirstRun = true;
        for (int i = 0; i < flist.length; i++) {
            Vector<String> row = new Vector<String>();
            Vector<String> examples = new Vector<String>();
            Vector<String> oexamples = new Vector<String>();
            if (!flist[i].getName().contains(".csv"))
                continue;
            CSVReader re = new CSVReader(new FileReader(flist[i]), '\t');
            String[] line = null;
            //discard the first line
            re.readNext();
            while ((line = re.readNext()) != null) {
                oexamples.add(line[0]);
                examples.add(line[1]);
            }
            RegularityFeatureSet rf = new RegularityFeatureSet();
            Collection<Feature> cf = rf.computeFeatures(oexamples, examples);
            //Iterator<Feature> iter = cf.iterator();
            Feature[] x = new Feature[cf.size()];
            cf.toArray(x);
            if (isfirstRun) {
                row.add("Featurename");
                for (int l = 0; l < x.length; l++) {
                    row.add(x[l].getName());
                }
                isfirstRun = false;
                row = new Vector<String>();
            }
            if (!isfirstRun) {
                row.add(flist[i].getName());
                for (int k = 0; k < cf.size(); k++) {
                    row.add(String.valueOf(x[k].getScore()));
                }
            }
        }
    } catch (Exception ex) {
        Logger.getLogger(Main.class).info("" + ex.toString());
    }
}

public ArrayList<Collection> getCollections() {
    ArrayList<Collection> collections = new ArrayList<Collection>();
    try {
        // create connection to AT
        logger.log("local string BAD!!", ATLogger.LogLevel.Debug);
        logger.log("Opening connection to AT Getting collections", ATLogger.LogLevel.Info);
        String uri = "http://www.academictorrents.com/collections.php?format=.csv";
        logger.log("uri: " + uri, ATLogger.LogLevel.Debug);
        URI collections_uri = new URI(uri);
        URLConnection collections_con = collections_uri.toURL().openConnection();
        // reader content from connection and create collection
        CSVReader reader = new CSVReader(new InputStreamReader(collections_con.getInputStream()));
        //skip csv header
        String[] line = reader.readNext();
        while ((line = reader.readNext()) != null) {
            Collection collection = new Collection(line[0], line[1], Integer.parseInt(line[2]), Long.parseLong(line[3]));
            collection.setTorrents(getCollectionEntries(line[1]));
            collections.add(collection);
            logger.log("Added collection to collections", ATLogger.LogLevel.Debug);
        }
    } catch (URISyntaxException e) {
        logger.log(e.getMessage() + e.getInput() + e.getReason() + e.getIndex(), ATLogger.LogLevel.Error);
    } catch (MalformedURLException e) {
        logger.log(e.getMessage(), ATLogger.LogLevel.Error);
    } catch (IOException e) {
        logger.log(e.getMessage(), ATLogger.LogLevel.Error);
    } catch (NumberFormatException e) {
        logger.log("Parse Error" + e.getMessage(), ATLogger.LogLevel.Error);
    } catch (Exception e) {
        logger.log("Unknown Exception in fetcher getting collections", ATLogger.LogLevel.Error);
    }
    return collections;
}

/**
     * @param args
     */
public static void main(String[] args) throws IOException {
    String[] ids;
    if (args.length == 2 || args.length == 3) {
        String server = "http://localhost:8888/";
        File csvFile = new File(args[0]);
        CSVReader csvReader;
        int index;
        if (!csvFile.exists()) {
            LOGGER.error(BUNDLE.get("TC_FILE_NOT_FOUND") + csvFile);
            printUsageAndExit();
        }
        // Make sure format of supplied server URL is what we expect
        if (args.length == 3) {
            if (!args[2].startsWith("http://")) {
                args[2] = "http://" + args[2];
            }
            if (!args[2].endsWith("/")) {
                args[2] = args[2] + "/";
            }
            server = args[2];
        }
        if (!isLive(server)) {
            LOGGER.error(BUNDLE.get("TC_SERVER_404"), server);
            printUsageAndExit();
        }
        try {
            csvReader = new CSVReader(new FileReader(csvFile));
            // columns 1-based
            index = Integer.parseInt(args[1]) - 1;
            while ((ids = csvReader.readNext()) != null) {
                if (LOGGER.isInfoEnabled()) {
                    LOGGER.info(BUNDLE.get("TC_CACHE_ID"), ids[index]);
                }
                cacheImage(server, ids[index]);
            }
        } catch (NumberFormatException details) {
            LOGGER.error(details.getMessage());
            printUsageAndExit();
        }
    } else // else if (args.length == 1) {} // TODO: descend through directories
    {
        printUsageAndExit();
    }
}

@Override
public HashMultimap<String, String> doGetAllReqCommitRelations() throws IOException {
    HashMultimap<String, String> relations = HashMultimap.create();
    try (CSVReader reader = getReader()) {
        List<String[]> allLines = reader.readAll();
        String reqId;
        String commitId;
        for (String[] line : allLines) {
            reqId = line[REQUIREMENT_COLUMN];
            commitId = line[COMMIT_COLUMN];
            relations.put(reqId, commitId);
        }
    }
    return relations;
}

/**
   * Extract the name of the toplevel corpus from the content of the
   * corpus.tab file.
   *
   * @param corpusTabContent
   * @return
   */
public static String extractToplevelCorpusNames(InputStream corpusTabContent) {
    String result = null;
    try {
        CSVReader csv = new CSVReader(new InputStreamReader(corpusTabContent, "UTF-8"), '\t');
        String[] line;
        int maxPost = Integer.MIN_VALUE;
        int minPre = Integer.MAX_VALUE;
        while ((line = csv.readNext()) != null) {
            if (line.length >= 6 && "CORPUS".equalsIgnoreCase(line[2])) {
                int pre = Integer.parseInt(line[4]);
                int post = Integer.parseInt(line[5]);
                if (pre <= minPre && post >= maxPost) {
                    minPre = pre;
                    maxPost = post;
                    result = line[1];
                }
            }
        }
    } catch (UnsupportedEncodingException ex) {
        log.error(null, ex);
    } catch (IOException ex) {
        log.error(null, ex);
    }
    return result;
}

public void addUserList(UserStoreManager userStore) throws UserAdminException {
    CSVReader csvReader = new CSVReader(reader, ',', '"', 1);
    try {
        String domain = config.getUserStoreDomain();
        String[] line = csvReader.readNext();
        boolean isDuplicate = false;
        boolean fail = false;
        boolean success = false;
        String lastError = "UNKNOWN";
        int successCount = 0;
        int failCount = 0;
        int duplicateCount = 0;
        while (line != null && line.length > 0) {
            String userName = line[0];
            int index;
            index = userName.indexOf(CarbonConstants.DOMAIN_SEPARATOR);
            if (index > 0) {
                String domainFreeName = userName.substring(index + 1);
                userName = UserCoreUtil.addDomainToName(domainFreeName, domain);
            } else {
                userName = UserCoreUtil.addDomainToName(userName, domain);
            }
            if (userName != null && userName.trim().length() > 0) {
                try {
                    if (!userStore.isExistingUser(userName)) {
                        if (line.length == 1) {
                            userStore.addUser(userName, null, null, null, null, true);
                            success = true;
                            successCount++;
                        } else {
                            try {
                                addUserWithClaims(userName, line, userStore);
                                success = true;
                                successCount++;
                                if (log.isDebugEnabled()) {
                                    log.debug("User import successful - Username : " + userName);
                                }
                            } catch (UserAdminException e) {
                                fail = true;
                                failCount++;
                                lastError = e.getMessage();
                                log.error("User import unsuccessful - Username : " + userName + " - Error: " + e.getMessage());
                            }
                        }
                    } else {
                        isDuplicate = true;
                        duplicateCount++;
                        log.error("User import unsuccessful - Username : " + userName + " - Error: Duplicate user");
                    }
                } catch (UserStoreException e) {
                    if (log.isDebugEnabled()) {
                        log.debug(e);
                    }
                    lastError = e.getMessage();
                    fail = true;
                    failCount++;
                    log.error("User import unsuccessful - Username : " + userName + " - Error: " + e.getMessage());
                }
            }
            line = csvReader.readNext();
        }
        log.info("Success count: " + successCount + ", Fail count: " + failCount + ", Duplicate count: " + duplicateCount);
        if (fail && success) {
            throw new UserAdminException("Error occurs while importing user names. " + "Success count: " + successCount + ", Fail count: " + failCount + ", Duplicate count: " + duplicateCount + ". Last error was : " + lastError);
        }
        if (fail && !success) {
            throw new UserAdminException("Error occurs while importing user names. " + "All user names were not imported. Last error was : " + lastError);
        }
        if (isDuplicate) {
            throw new UserAdminException("Detected " + duplicateCount + " duplicate user names. " + "Failed to import duplicate users. Non-duplicate user names were successfully imported.");
        }
    } catch (UserAdminException e) {
        throw e;
    } catch (Exception e) {
        log.error("Error occured while adding userlist", e);
        throw new UserAdminException("Error occured while adding userlist", e);
    } finally {
        try {
            if (csvReader != null) {
                csvReader.close();
            }
        } catch (IOException e) {
            log.error("Error occurred while closing CSV Reader", e);
        }
    }
}

private void createDataModel() throws IOException {
    CSVReader reader = new CSVReader(new FileReader(properties.getProperty("mahout.dumpfile")));
    // Read header
    reader.readNext();
    Map<Long, List<Preference>> preferences = new HashMap<Long, List<Preference>>();
    Long userId = -1L;
    Long itemId = -1L;
    while (true) {
        String[] nextLine = reader.readNext();
        if (nextLine == null) {
            break;
        }
        String item = nextLine[0];
        String title = nextLine[1];
        String user = nextLine[2];
        if (!userToId.containsKey(user)) {
            userId++;
            userToId.put(user, userId);
            idToUser.put(userId, user);
        }
        if (!itemToId.containsKey(item)) {
            itemId++;
            ChannelData chData = new ChannelData();
            chData.setId(item);
            chData.setTitle(title);
            itemToId.put(item, itemId);
            idToItem.put(itemId, chData);
        }
        Long currentUserId = userToId.get(user);
        BooleanPreference booleanPreference = new BooleanPreference(currentUserId, itemToId.get(item));
        List<Preference> prefList = preferences.get(currentUserId);
        if (prefList == null) {
            prefList = new LinkedList<Preference>();
            preferences.put(currentUserId, prefList);
        }
        prefList.add(booleanPreference);
    }
    FastByIDMap<PreferenceArray> userData = new FastByIDMap<PreferenceArray>();
    for (Entry<Long, List<Preference>> entry : preferences.entrySet()) {
        userData.put(entry.getKey(), new BooleanUserPreferenceArray(entry.getValue()));
    }
    this.dataModel = new GenericDataModel(userData);
}

public Object createEntry(Object key) throws Exception {
    LocationBean location = null;
    try {
        // Requested zip code
        String zipCodeToFind = String.valueOf(key);
        // Data source
        CSVReader reader = new CSVReader(new InputStreamReader(LocationUtils.class.getResourceAsStream("/zipcode.csv")));
        //"zip","city","state","latitude","longitude","timezone","dst"
        //"00210","Portsmouth","NH","43.005895","-71.013202","-5","1"
        // Get the header columns
        String[] nextLine = reader.readNext();
        int zipColumn = 0;
        int cityColumn = 1;
        int stateColumn = 2;
        int latitudeColumn = 3;
        int longitudeColumn = 4;
        int timeZoneColumn = 5;
        int dstColumn = 6;
        // find the zipcode
        while ((nextLine = reader.readNext()) != null) {
            if (nextLine.length < 7) {
                continue;
            }
            String zipValue = nextLine[zipColumn].trim();
            if (zipCodeToFind.equals(zipValue)) {
                location = new LocationBean();
                location.setZipCode(nextLine[zipColumn]);
                location.setCity(nextLine[cityColumn]);
                location.setState(nextLine[stateColumn]);
                location.setLatitude(nextLine[latitudeColumn]);
                location.setLongitude(nextLine[longitudeColumn]);
                location.setTimeZone(nextLine[timeZoneColumn]);
                location.setDst(nextLine[dstColumn]);
                break;
            }
        }
        reader.close();
    } catch (Exception e) {
        throw new Exception(e);
    }
    return location;
}

/**
     *
     * @param input
     * @param csvSeparatorChar
     */
public void setCsvColumnList(CorpusData input, char csvSeparatorChar) {
    List<FileData> files = input.getFiles();
    List<String> csvColumnListData = new ArrayList<String>();
    try {
        FileData firstFileData = files.get(0);
        String firstFileName = firstFileData.getFile().getPath();
        CSVReader reader = new CSVReader(new FileReader(firstFileName), csvSeparatorChar);
        String[] nextLine = reader.readNext();
        if (nextLine != null) {
            csvColumnListData = Arrays.asList(nextLine);
        }
        this.csvColumnList = csvColumnListData;
        reader.close();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

@Override
public void run() {
    ArrayList<String> schema;
    // and construct a schema for this csv file.
    try {
        BufferedReader bufReader = new BufferedReader(new FileReader(csvData));
        String schemaLine = bufReader.readLine();
        String[] schemaEntries = StringUtils.split(schemaLine, Defaults.COMMA);
        schema = Lists.newArrayListWithCapacity(schemaEntries.length);
        for (String schemaEntry : schemaEntries) {
            schema.add(StringUtils.trim(schemaEntry));
        }
        bufReader.close();
    } catch (FileNotFoundException e) {
        throw new RuntimeException(e);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    CSVReader reader = null;
    try {
        reader = new CSVReader(new FileReader(csvData));
        // Consume the schema line as we don't want to treat it as a record
        reader.readNext();
        String[] data;
        long lineNumber = 0l;
        long resultsInserted = 0l;
        int numFlushes = 0;
        int logUpdateEveryNBuffers = 5;
        int itemsToBuffer = 10000;
        long entriesLoadedSinceLastLog = 0l;
        // Buffer 100 results to ammortize the BatchWriter costs underneath
        ArrayList<MultimapRecord> cachedResults = Lists.newArrayListWithCapacity(itemsToBuffer + 1);
        while (null != (data = reader.readNext())) {
            lineNumber++;
            // Make sure our line of data has the expected number of columns
            if (data.length != schema.size()) {
                log.error("Data on line {} did not match expected column size: {}", lineNumber, data);
                continue;
            }
            // Make the Multimap of column to svalue
            HashMultimap<Column, RecordValue<?>> metadata = HashMultimap.create();
            for (int i = 0; i < schema.size(); i++) {
                String name = schema.get(i);
                String value = StringUtils.trim(data[i]);
                if (!StringUtils.isBlank(value)) {
                    metadata.put(Column.create(name), RecordValue.create(value, Defaults.EMPTY_VIS));
                }
            }
            // Make sure we can extract a column as the docID
            if (!metadata.containsKey(ID) || 1 != metadata.get(ID).size()) {
                log.error("Expected to find one {} column in record: {}", ID, metadata.toString());
            }
            RecordValue<?> docId = metadata.get(ID).iterator().next();
            // Add the record to our buffer
            cachedResults.add(new MultimapRecord(metadata, docId.value().toString(), Defaults.EMPTY_VIS));
            // Flush the buffer when it gets big enough
            if (itemsToBuffer < cachedResults.size()) {
                try {
                    cosmos.addResults(this.id, cachedResults);
                    numFlushes++;
                    entriesLoadedSinceLastLog += cachedResults.size();
                } catch (Exception e) {
                    log.error("Problem adding results to cosmos", e);
                    throw new RuntimeException(e);
                }
                if (0 == numFlushes % logUpdateEveryNBuffers) {
                    log.info("Loaded {} records", entriesLoadedSinceLastLog);
                    numFlushes = 0;
                    entriesLoadedSinceLastLog = 0l;
                }
                resultsInserted += cachedResults.size();
                cachedResults.clear();
            }
            // Observe a limit on how many results we'll load after each batch
            if (resultsInserted >= this.maxResultsToLoad) {
                break;
            }
        }
        if (0 < numFlushes) {
            log.info("Loaded {} records", entriesLoadedSinceLastLog);
        }
        // Make sure we catch the tail-end of any results we buffered
        if (!cachedResults.isEmpty()) {
            try {
                cosmos.addResults(this.id, cachedResults);
                log.info("Loaded {} records", cachedResults.size());
                // Try to force a cleanup on next GC
                cachedResults = null;
            } catch (Exception e) {
                log.error("Problem adding results to cosmos", e);
                throw new RuntimeException(e);
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    } finally {
        if (null != reader) {
            try {
                reader.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
}

/**
     * Data loader
     * @param strFilePath
     * @param AssocId
     * @param Simple
     * @return
     * @throws FileNotFoundException
     */
public static String LoadDataAssociations(String strFilePath, Integer AssocId, Boolean Simple) throws FileNotFoundException {
    Map<java.lang.Integer, java.lang.Double> UserCompare = new TreeMap<java.lang.Integer, java.lang.Double>();
    @SuppressWarnings("unused") Collection<Integer> advUserCompare = new ArrayList<Integer>();
    Collection<ratingsData> UserRatings = new ArrayList<ratingsData>();
    String output = AssocId.toString();
    String adVoutput = AssocId.toString();
    String result = "";
    Integer xbarValue = 0;
    try {
        @SuppressWarnings("unused") int ii = 0;
        //Read the input file and convert into
        @SuppressWarnings("resource") CSVReader reader = new CSVReader(new FileReader(strFilePath), ',');
        String[] nextLine;
        while ((nextLine = reader.readNext()) != null) {
            ratingsData ratingsDataVal = new ratingsData();
            //Comparing MovieId with AssocId
            if (java.lang.Integer.parseInt(nextLine[1]) == AssocId) {
                UserCompare.put(java.lang.Integer.parseInt(nextLine[0]), java.lang.Double.parseDouble(nextLine[2]));
            } else {
                ratingsDataVal.setUserId(java.lang.Integer.parseInt(nextLine[0]));
                ratingsDataVal.setMovieId(java.lang.Integer.parseInt(nextLine[1]));
                ratingsDataVal.setRatingValue(java.lang.Double.parseDouble(nextLine[2]));
                UserRatings.add(ratingsDataVal);
            }
        }
        Map<java.lang.Integer, java.lang.Integer> MovieRatings = new TreeMap<java.lang.Integer, java.lang.Integer>();
        //Y Values
        MovieRatings = calculate_ratings.findYvalue(UserRatings, UserCompare);
        //simple association
        Map<java.lang.Integer, java.lang.Double> MovieAssociations = new TreeMap<java.lang.Integer, java.lang.Double>();
        //simple association
        Map<java.lang.Integer, java.lang.Double> AdvMovieAssociations = new TreeMap<java.lang.Integer, java.lang.Double>();
        //Advanced Value
        Map<java.lang.Integer, java.lang.Integer> advMovieRatings = new TreeMap<java.lang.Integer, java.lang.Integer>();
        //findXBarvalue
        advMovieRatings = calculate_ratings.findXBarvalue(UserRatings, UserCompare);
        xbarValue = calculate_ratings.XBarvalue(UserRatings, UserCompare);
        for (Iterator<java.lang.Integer> mr = MovieRatings.keySet().iterator(); mr.hasNext(); ) {
            //(x+y)/y
            java.lang.Integer dmr = mr.next();
            java.lang.Double Val = (double) (MovieRatings.get(dmr) + UserCompare.size()) / UserCompare.size();
            MovieAssociations.put(dmr, (Val - 1));
            Float advValue = (float) ((Val - 1) / (((float) (advMovieRatings.get(dmr)) / (xbarValue))));
            AdvMovieAssociations.put(dmr, (double) advValue);
        }
        //Sorted - Simple
        Map<java.lang.Integer, java.lang.Double> SortedMovieAssociations = new TreeMap<java.lang.Integer, java.lang.Double>();
        SortedMovieAssociations = sort_map.sortMapByValue(MovieAssociations);
        int h = 0;
        for (Iterator<java.lang.Integer> smr = SortedMovieAssociations.keySet().iterator(); smr.hasNext(); ) {
            h = h + 1;
            java.lang.Integer dsmr = smr.next();
            output = output + "," + dsmr + "," + SortedMovieAssociations.get(dsmr).toString().substring(0, 5);
            if (h >= recsys_code_pa1.TOP_5) {
                break;
            }
        }
        //Sorted - Advanced
        Map<java.lang.Integer, java.lang.Double> SortedAdvMovieAssociations = new TreeMap<java.lang.Integer, java.lang.Double>();
        SortedAdvMovieAssociations = sort_map.sortMapByValue(AdvMovieAssociations);
        int adv = 0;
        for (Iterator<java.lang.Integer> asmr = SortedAdvMovieAssociations.keySet().iterator(); asmr.hasNext(); ) {
            adv = adv + 1;
            java.lang.Integer adsmr = asmr.next();
            //output
            adVoutput = adVoutput + "," + adsmr + "," + SortedAdvMovieAssociations.get(adsmr).toString().substring(0, 5);
            //Top 5
            if (adv >= recsys_code_pa1.TOP_5) {
                break;
            }
        }
        if (Simple == true) {
            result = adVoutput;
        } else {
            result = output;
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
    }
    return result;
}

public void importCSV(File unigrams, File bigrams, File trigrams) throws IOException, SQLException {
    CSVReader csvReaderUnigram = new CSVReader(new FileReader(unigrams));
    String[] line;
    PreparedStatement unigramInsert = this.sqlConnection.prepareStatement("INSERT INTO words VALUES (?, ?, ?, ?);");
    while ((line = csvReaderUnigram.readNext()) != null) {
        if (line.length != 4)
            break;
        unigramInsert.setInt(1, Integer.parseInt(line[0]));
        unigramInsert.setString(2, line[1]);
        unigramInsert.setLong(3, line[2].length() == 0 ? 0 : Long.parseLong(line[2]));
        unigramInsert.setLong(4, line[3].length() == 0 ? 0 : Long.parseLong(line[3]));
        unigramInsert.addBatch();
    }
    unigramInsert.executeBatch();
    CSVReader csvReaderBigrams = new CSVReader(new FileReader(bigrams));
    PreparedStatement bigramInsert = this.sqlConnection.prepareStatement("INSERT INTO bigrams VALUES (?, ?, ?, ?, ?, ?);");
    while ((line = csvReaderBigrams.readNext()) != null) {
        if (line.length != 6)
            break;
        float significance_web = Float.parseFloat(line[5]);
        if (significance_web < Math.max(BIGRAM_LEFT_MIN_SIGNIFICANCE_WEB, BIGRAM_RIGHT_MIN_SIGNIFICANCE_WEB))
            continue;
        bigramInsert.setInt(1, Integer.parseInt(line[0]));
        bigramInsert.setInt(2, Integer.parseInt(line[1]));
        bigramInsert.setInt(3, Integer.parseInt(line[2]));
        bigramInsert.setFloat(4, Float.parseFloat(line[3]));
        bigramInsert.setLong(5, Long.parseLong(line[4]));
        bigramInsert.setFloat(6, significance_web);
        bigramInsert.addBatch();
    }
    bigramInsert.executeBatch();
    CSVReader csvReaderTrigrams = new CSVReader(new FileReader(trigrams));
    PreparedStatement trigramInsert = this.sqlConnection.prepareStatement("INSERT INTO trigrams VALUES (?, ?, ?, ?);");
    while ((line = csvReaderTrigrams.readNext()) != null) {
        if (line.length != 4)
            break;
        long count_web = Long.parseLong(line[3]);
        if (count_web < Math.max(TRIGRAM_LEFT_MIN_COUNT_WEB, TRIGRAM_RIGHT_MIN_COUNT_WEB)) {
            continue;
        }
        trigramInsert.setInt(1, Integer.parseInt(line[0]));
        trigramInsert.setInt(2, Integer.parseInt(line[1]));
        trigramInsert.setInt(3, Integer.parseInt(line[2]));
        trigramInsert.setLong(4, count_web);
        trigramInsert.addBatch();
    }
    trigramInsert.executeBatch();
    sqlConnection.close();
}

private Result<String> importCSV(final String file, final IProgressMonitor mon) {
    try {
        CSVReader cr = new CSVReader(new FileReader(file));
        String[] line;
        while ((line = cr.readNext()) != null) {
            importLine(line);
        }
        //$NON-NLS-1$
        return new Result<String>("OK");
    } catch (Exception ex) {
        ExHandler.handle(ex);
        return new Result<String>(Result.SEVERITY.ERROR, 1, Messages.Eigendiagnosen_CantRead + file, ex.getMessage(), true);
    }
}