Java Examples for org.apache.commons.csv.CSVFormat

@Override
public void configure() throws Exception {
    // get Report SObject by DeveloperName
    from("direct:queryReport").to("salesforce:query?sObjectClass=" + QueryRecordsReport.class.getName());
    from("direct:getRecentReports").to("salesforce:getRecentReports");
    from("direct:getReportDescription").to("salesforce:getReportDescription");
    from("direct:executeSyncReport").to("salesforce:executeSyncReport");
    from("direct:executeAsyncReport").to("salesforce:executeAsyncReport?includeDetails=true");
    from("direct:getReportInstances").to("salesforce:getReportInstances");
    from("direct:getReportResults").to("salesforce:getReportResults");
    CsvDataFormat csv = new CsvDataFormat(CSVFormat.EXCEL);
    // type converter test
    from("direct:convertResults").convertBodyTo(List.class).marshal(csv);
}

/**
     * To CSV.
     *
     * @param outputCSVFileName the output CSV file name
     * @param results the list of results
     * @param fields the list of fields
     * @param <ResultT> the result type
     */
public static <ResultT> void toCsv(String outputCSVFileName, List<ResultT> results, List<CSVField<ResultT>> fields) {
    File resultDirFile = new File(outputCSVFileName).getParentFile();
    if (!resultDirFile.exists()) {
        if (!resultDirFile.mkdirs()) {
            throw new RuntimeException();
        }
    }
    CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(',').withQuoteMode(QuoteMode.ALL).withQuote('"');
    try {
        FileWriter writer = new FileWriter(outputCSVFileName);
        CSVPrinter printer = new CSVPrinter(writer, format);
        printer.printRecord(fields);
        for (ResultT result : results) {
            List<String> record = new ArrayList<>();
            for (CSVField<ResultT> field : fields) {
                record.add(field.getFunc().apply(result));
            }
            printer.printRecord(record);
        }
        printer.close();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

@Override
public void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {
    ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes());
    int n = value.getLength();
    byte[] bytes = new byte[n];
    inputStream.read(bytes, 0, n);
    CSVParser parser = CSVParser.parse(new String(bytes), CSVFormat.DEFAULT);
    try {
        for (CSVRecord csvRecord : parser) {
            for (String string : csvRecord) {
                word.set(string);
                context.write(word, one);
                Counter counter = context.getCounter(CountersEnum.class.getName(), CountersEnum.INPUT_WORDS.toString());
                counter.increment(1);
            }
        }
    } catch (Exception e) {
    }
}

public void write(List<Patient> patients, String file) {
    CSVFormat csvFileFormat = CSVFormat.EXCEL;
    try {
        StringBuilder builder = new StringBuilder();
        CSVPrinter csvPrinter = new CSVPrinter(builder, csvFileFormat);
        csvPrinter.printRecord(PatientCSVConverter.FILE_HEADER);
        for (Patient patient : patients) {
            csvPrinter.printRecord(new PatientCSVConverter().convert(patient));
        }
        FileWriter writer = new FileWriter(file);
        writer.write(builder.toString());
        writer.flush();
    } catch (IOException e) {
        throw new RuntimeException("Cannot generate csv file", e);
    }
}

private static void runTest(File source, File gold, File expectations, String command) throws Exception {
    FileReader in = new FileReader(gold);
    Iterable<CSVRecord> records = CSVFormat.RFC4180.withHeader().parse(in);
    Set<String> expected = new HashSet<>();
    for (CSVRecord record : records) {
        expected.add(record.get("identifier"));
    }
    final byte[] jsonData = Files.readAllBytes(expectations.toPath());
    JSONObject jsonObject = (JSONObject) JSONSerializer.toJSON(new String(jsonData));
    JSONObject identifiers = jsonObject.getJSONObject("identifiers");
    double expPrec = identifiers.getDouble("precision");
    double expRec = identifiers.getDouble("recall");
    String[] args = { command, "-in", source.getAbsolutePath(), "--tex" };
    final PrintStream stdout = System.out;
    final ByteArrayOutputStream myOut = new ByteArrayOutputStream();
    System.setOut(new PrintStream(myOut));
    final long t0 = System.nanoTime();
    Main.main(args);
    final String standardOutput = myOut.toString();
    System.setOut(stdout);
    System.out.println((System.nanoTime() - t0) / 1000000000 + "s");
    Set<String> sysOut = new HashSet<>(Arrays.asList(standardOutput.split("\r?\n")));
    Set<String> real = new HashSet<>();
    for (String s : sysOut) {
        real.add(s.replaceAll("_\\{(.*?)\\}$", "_"));
    }
    Set<String> tp = new HashSet<>(expected);
    Set<String> fn = new HashSet<>(expected);
    Set<String> fp = new HashSet<>(real);
    fn.removeAll(real);
    fp.removeAll(expected);
    tp.retainAll(real);
    double rec = ((double) tp.size()) / (tp.size() + fn.size());
    double prec = ((double) tp.size()) / (tp.size() + fp.size());
    assertThat("precision", prec, Matchers.greaterThan(expPrec));
    assertThat("recall", rec, Matchers.greaterThan(expRec));
}

private static List<String[]> getRecords(String path) {
    List<String[]> records = new ArrayList<String[]>();
    try {
        Reader reader = new FileReader(BigFileAnalyzerPerformanceTest.class.getResource(path).getPath());
        CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(';').withFirstRecordAsHeader();
        Iterable<CSVRecord> csvRecords = csvFormat.parse(reader);
        for (CSVRecord csvRecord : csvRecords) {
            String[] values = new String[csvRecord.size()];
            for (int i = 0; i < csvRecord.size(); i++) {
                values[i] = csvRecord.get(i);
            }
            records.add(values);
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return records;
}

public static void main(String[] args) throws IOException {
    File csvData = new File("khan.csv");
    CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8, CSVFormat.RFC4180);
    PrintStream out = new PrintStream("khan.sql");
    int registro = 1;
    for (CSVRecord csvRecord : parser) {
        if (registro++ == 1)
            continue;
        out.print("insert into khan (");
        out.print("rede,escola,turma,login,aluno,com_dificuldade,precisa_praticar,praticado,nivel1,nivel2,dominado,pontos,exerciseminutes,videominutes,totalminutes,semana");
        out.print(") values (");
        out.print("'" + limpaTexto(csvRecord.get(0)) + "',");
        out.print("'" + limpaTexto(csvRecord.get(1)) + "',");
        out.print("'" + limpaTexto(csvRecord.get(2).replace("'", "")) + "',");
        out.print("'" + limpaTexto(csvRecord.get(3)) + "',");
        out.print("'" + limpaTexto(csvRecord.get(4)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(5)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(6)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(7)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(8)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(9)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(10)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(11)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(12)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(13)) + "',");
        out.print("'" + limpaNumero(csvRecord.get(14)) + "',");
        out.print("'" + csvRecord.get(15) + "')");
        out.print(";");
        out.println();
    }
    out.close();
}

public String invoke(String sender, String messageContent) {
    try {
        String command = messageContent;
        List<String> commandArgs = new ArrayList<>();
        if (messageContent.contains(" ")) {
            command = substringBefore(messageContent, " ");
            String args = substringAfter(messageContent, " ");
            CSVParser csvRecords = CSVParser.parse(args, CSVFormat.newFormat(' ').withQuote('"'));
            commandArgs = Lists.newArrayList(csvRecords.iterator().next().iterator());
            commandArgs = commandArgs.stream().map(( arg) -> arg.startsWith("$") ? store.get(sender, substringAfter(arg, "$"), String.class) : arg).collect(Collectors.toList());
        }
        return commands.get(command).invoke(sender, commandArgs);
    } catch (Exception ex) {
        throw new JanuszException(ex);
    }
}

@Override
protected void init(Writer writer) {
    CsvFormatBuilder builder = new CsvFormatBuilder(this);
    CSVFormat format = builder.build();
    doubleQuote = valueOf(builder.getQuote()) + valueOf(builder.getQuote());
    try {
        csvPrinter = new CSVPrinter(wrapWriter(writer), format);
    } catch (IOException exception) {
        throw new OutputException(exception);
    }
}

/**
     * Default implementation uses a CSV parser to extract the first value, then sets the raw line of text on the summary
     * to make it available to other plugins. Note that any overriding implementation *must* catch Exception from
     * {@link #createSummary(String)} and return a new zero-sized {@link ObjectSummary} for the identifier
     */
@Override
public ObjectSummary parseListLine(String listLine) {
    try {
        CSVRecord record = CSVFormat.EXCEL.parse(new StringReader(listLine)).iterator().next();
        ObjectSummary summary;
        try {
            summary = createSummary(record.get(0));
        } catch (Exception e) {
            summary = new ObjectSummary(record.get(0), false, 0);
        }
        summary.setListFileRow(listLine);
        return summary;
    } catch (IOException e) {
        throw new RuntimeException("could not parse list-file line", e);
    }
}

@Test
public void exportToCsv() throws E, IOException {
    final R queryRequest = queryRequestBuilderFactory.getObject().queryRestrictions(testUtils.buildQueryRestrictions()).queryType(QueryRequest.QueryType.MODIFIED).build();
    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    exportService.exportQueryResults(outputStream, queryRequest, ExportFormat.CSV, Collections.emptyList(), 1001L);
    final String output = outputStream.toString();
    assertNotNull(output);
    try (final CSVParser csvParser = CSVParser.parse(output, CSVFormat.EXCEL)) {
        final List<CSVRecord> records = csvParser.getRecords();
        assertThat(records, not(empty()));
        final CSVRecord headerRecord = records.get(0);
        // byte-order mark may get in the way
        assertThat(headerRecord.get(0), endsWith("Reference"));
        assertEquals("Database", headerRecord.get(1));
        final CSVRecord firstDataRecord = records.get(1);
        final String firstDataRecordReference = firstDataRecord.get(0);
        assertNotNull(firstDataRecordReference);
        assertFalse(firstDataRecordReference.trim().isEmpty());
        final String firstDataRecordDatabase = firstDataRecord.get(1);
        assertFalse(firstDataRecordDatabase.trim().isEmpty());
    }
}

@Test
public void testHeaderFooter() {
    final String header = "# Header";
    final String footer = "# Footer ";
    final AbstractCsvLayout layout = CsvLogEventLayout.createLayout(ctx.getConfiguration(), "Excel", null, null, null, null, null, null, null, header, footer);
    testLayout(CSVFormat.DEFAULT, layout, header, footer);
}

/**
     * {@inheritDoc}
     */
@Override
public TemporalDataModelIF<Long, Long> parseTemporalData(final File f) throws IOException {
    TemporalDataModelIF<Long, Long> dataset = new TemporalDataModel<>();
    Reader in = new InputStreamReader(new FileInputStream(f), "UTF-8");
    Iterable<CSVRecord> records;
    if (isHasHeader()) {
        records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).withHeader().parse(in);
    } else {
        records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).parse(in);
    }
    for (CSVRecord record : records) {
        long userID = Long.parseLong(record.get(getUserTok()));
        long itemID = Long.parseLong(record.get(getItemTok()));
        long timestamp = -1L;
        if (getTimeTok() != -1) {
            timestamp = Long.parseLong(record.get(getTimeTok()));
        }
        double preference = Double.parseDouble(record.get(getPrefTok()));
        dataset.addPreference(userID, itemID, preference);
        dataset.addTimestamp(userID, itemID, timestamp);
    }
    in.close();
    return dataset;
}

@Test
public void testFindSpreadsheetsFromCells() throws IOException {
    CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestSpreadsheetExtractor-CELLS.csv"), Charset.forName("utf-8"), CSVFormat.DEFAULT);
    List<Cell> cells = new ArrayList<Cell>();
    for (CSVRecord record : parse) {
        cells.add(new Cell(Float.parseFloat(record.get(0)), Float.parseFloat(record.get(1)), Float.parseFloat(record.get(2)), Float.parseFloat(record.get(3))));
    }
    SpreadsheetExtractionAlgorithm se = new SpreadsheetExtractionAlgorithm();
    List<Rectangle> expected = Arrays.asList(EXPECTED_RECTANGLES);
    Collections.sort(expected);
    List<Rectangle> foundRectangles = se.findSpreadsheetsFromCells(cells);
    Collections.sort(foundRectangles);
    assertTrue(foundRectangles.equals(expected));
}

private DbSettings getTargetDbSettings() {
    DbSettings dbSettings = new DbSettings();
    if (targetType.getSelectedItem().equals("Delimited text files")) {
        dbSettings.dataType = DbSettings.CSVFILES;
        switch((String) targetCSVFormat.getSelectedItem()) {
            case "Default (comma, CRLF)":
                dbSettings.csvFormat = CSVFormat.DEFAULT;
                break;
            case "RFC4180":
                dbSettings.csvFormat = CSVFormat.RFC4180;
                break;
            case "Excel CSV":
                dbSettings.csvFormat = CSVFormat.EXCEL;
                break;
            case "TDF (tab, CRLF)":
                dbSettings.csvFormat = CSVFormat.TDF;
                break;
            case "MySQL (tab, LF)":
                dbSettings.csvFormat = CSVFormat.MYSQL;
                break;
            default:
                dbSettings.csvFormat = CSVFormat.RFC4180;
        }
    } else {
        dbSettings.dataType = DbSettings.DATABASE;
        dbSettings.user = targetUserField.getText();
        dbSettings.password = targetPasswordField.getText();
        dbSettings.server = targetServerField.getText();
        dbSettings.database = targetDatabaseField.getText();
        if (targetType.getSelectedItem().toString().equals("MySQL"))
            dbSettings.dbType = DbType.MYSQL;
        else if (targetType.getSelectedItem().toString().equals("Oracle"))
            dbSettings.dbType = DbType.ORACLE;
        else if (sourceType.getSelectedItem().toString().equals("PostgreSQL"))
            dbSettings.dbType = DbType.POSTGRESQL;
        else if (sourceType.getSelectedItem().toString().equals("SQL Server")) {
            dbSettings.dbType = DbType.MSSQL;
            if (// Not using windows authentication
            sourceUserField.getText().length() != 0) {
                String[] parts = sourceUserField.getText().split("/");
                if (parts.length == 2) {
                    dbSettings.user = parts[1];
                    dbSettings.domain = parts[0];
                }
            }
        }
        if (dbSettings.database.trim().length() == 0) {
            String message = "Please specify a name for the target database";
            JOptionPane.showMessageDialog(frame, StringUtilities.wordWrap(message, 80), "Database error", JOptionPane.ERROR_MESSAGE);
            return null;
        }
    }
    return dbSettings;
}

@Override
public void write(OutputStream os) throws IOException, WebApplicationException {
    Writer writer = new BufferedWriter(new OutputStreamWriter(os));
    CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT);
    try {
        List<ColumnDescription> descriptions = resultSet.getDescriptions();
        List<String> headers = Lists.newArrayList();
        for (ColumnDescription description : descriptions) {
            headers.add(description.getName());
        }
        csvPrinter.printRecord(headers.toArray());
        while (resultSet.hasNext()) {
            csvPrinter.printRecord(resultSet.next().getRow());
            writer.flush();
        }
    } finally {
        writer.close();
    }
}

/**
     * Parses CSV data
     * 
     * @param csvHeader Header items for csv records
     * @param csvData
     * @return
     */
public static List<SpeedTestRecord> parseCsvData(String csvHeader, String csvData) {
    Reader in = new StringReader(getCsvData(csvHeader, csvData));
    try {
        CSVParser parser = new CSVParser(in, CSVFormat.DEFAULT.toBuilder().withHeader().build());
        // get the parsed records
        List<CSVRecord> list = parser.getRecords();
        // create a list to convert data into SpeedTestRecord model
        List<SpeedTestRecord> speedTestRecords = new ArrayList<SpeedTestRecord>();
        for (CSVRecord csvRecord : list) {
            speedTestRecords.add(new SpeedTestRecord(csvRecord));
        }
        return speedTestRecords;
    } catch (IOException e) {
        Tracer.error(LOG_TAG, e);
    }
    // when no data, send empty list
    return Collections.emptyList();
}

@Override
public void Write(AnalyzerResult result, String _csvPath, int _counter) {
    try {
        ArrayList<AnalyzerResult> resultList = new ArrayList<AnalyzerResult>();
        resultList.add(result);
        /*--Create the CSVFormat object--*/
        CSVFormat format = CSVFormat.DEFAULT.withHeader();
        /*--Writing in a CSV file--*/
        FileWriter _out = new FileWriter(_csvPath, true);
        CSVPrinter printer;
        printer = new CSVPrinter(_out, format.withDelimiter('#'));
        /*--Retrieve APKResult and Write in file--*/
        Iterator<AnalyzerResult> it = resultList.iterator();
        while (it.hasNext()) {
            AnalyzerResult _resultElement = it.next();
            List<String> resultData = new ArrayList<>();
            resultData.add(String.valueOf(_counter));
            resultData.add(_resultElement.get_APKName());
            resultData.add(_resultElement.get_APKPath());
            resultData.add(_resultElement.get_Package());
            resultData.add(_resultElement.get_APKMainFramework());
            resultData.add(_resultElement.get_APKBaseFramework());
            resultData.add(String.valueOf(_resultElement.get_html()));
            resultData.add(String.valueOf(_resultElement.get_js()));
            resultData.add(String.valueOf(_resultElement.get_css()));
            resultData.add(_resultElement.get_debuggable());
            resultData.add(_resultElement.get_permissions());
            resultData.add(_resultElement.get_minSdkVersion());
            resultData.add(_resultElement.get_maxSdkVersion());
            resultData.add(_resultElement.get_targetSdkVersion());
            resultData.add(_resultElement.get_fileSize());
            resultData.add(String.valueOf(_resultElement.get_startAnalysis()));
            resultData.add(String.valueOf(_resultElement.get_durationAnalysis()));
            resultData.add(String.valueOf(_resultElement.get_decodeSuccess()));
            printer.printRecord(resultData);
        }
        /*--Close the printer--*/
        printer.close();
        System.out.println("Record added to CSV file");
        this.removeBlankLines(_csvPath);
    } catch (IOException ex) {
        Logger.getLogger(WriterCSV.class.getName()).log(Level.SEVERE, null, ex);
    }
}

public static void main(String[] args) throws IOException, NumberFormatException, InterruptedException {
    if (args.length < 4) {
        log.info("Usage: \n param 1 - path to file \n param 2 - stream name to send the data \n param 3 - time in ms to wait to send each data \n param 4 - broker list");
    } else {
        Producer<String, String> producer = new Producer<String, String>(createProducerConfig(args[3]));
        Gson gson = new Gson();
        Reader in = new FileReader(args[0]);
        CSVParser parser = CSVFormat.DEFAULT.parse(in);
        List<String> columnNames = new ArrayList<>();
        for (CSVRecord csvRecord : parser.getRecords()) {
            if (columnNames.size() == 0) {
                Iterator<String> iterator = csvRecord.iterator();
                while (iterator.hasNext()) {
                    columnNames.add(iterator.next());
                }
            } else {
                StratioStreamingMessage message = new StratioStreamingMessage();
                message.setOperation(STREAM_OPERATIONS.MANIPULATION.INSERT.toLowerCase());
                message.setStreamName(args[1]);
                message.setTimestamp(System.currentTimeMillis());
                message.setSession_id(String.valueOf(System.currentTimeMillis()));
                message.setRequest_id(String.valueOf(System.currentTimeMillis()));
                message.setRequest("dummy request");
                List<ColumnNameTypeValue> sensorData = new ArrayList<>();
                for (int i = 0; i < columnNames.size(); i++) {
                    // Workaround
                    Object value = null;
                    try {
                        value = Double.valueOf(csvRecord.get(i));
                    } catch (NumberFormatException e) {
                        value = csvRecord.get(i);
                    }
                    sensorData.add(new ColumnNameTypeValue(columnNames.get(i), null, value));
                }
                message.setColumns(sensorData);
                String json = gson.toJson(message);
                log.info("Sending data: {}", json);
                producer.send(new KeyedMessage<String, String>(InternalTopic.TOPIC_DATA.getTopicName(), STREAM_OPERATIONS.MANIPULATION.INSERT, json));
                log.info("Sleeping {} ms...", args[2]);
                Thread.sleep(Long.valueOf(args[2]));
            }
        }
        log.info("Program completed.");
    }
}

public void submitPosts() throws IOException, InterruptedException {
    List<String> names = loadNames();
    for (String name : names) {
        people.save(new Person(name));
    }
    Random random = new Random();
    Reader in = new FileReader(new File("data", "/Sentiment Analysis Dataset.csv"));
    // Some of the data has comments. Don't use any comment characters or escape
    // characters
    CSVParser tweets = CSVFormat.newFormat(',').withHeader().parse(in);
    int tweetCount = 0;
    for (CSVRecord tweet : tweets) {
        String name = names.get(random.nextInt(names.size()));
        String post = tweet.get("SentimentText");
        posts.save(new Post(name, post));
        tweetCount++;
        System.out.println(name + ": " + post);
        Thread.sleep(1000);
        if (System.in.available() > 0) {
            System.out.println("Exiting...");
            System.out.println("Posted " + tweetCount + " tweets");
            break;
        }
    }
}

public List<CSVRecord> readFromCSVFile(String filePath) throws IOException {
    CSVFormat format = CSVFormat.RFC4180.withDelimiter(',').withCommentMarker('#');
    try (InputStream inputStream = getClass().getResourceAsStream(filePath);
        CSVParser parser = new CSVParser(new InputStreamReader(inputStream), format)) {
        List<CSVRecord> csvRecords = parser.getRecords();
        return csvRecords;
    }
}

private void writeToCsv(Product product) throws IOException {
    // TODO improve
    if (csvPrinter == null) {
        CSVFormat csvFormat = CSVFormat.RFC4180.withHeader().withDelimiter(',');
        csvPrinter = new CSVPrinter(new FileWriter(CSV_FILE), csvFormat.withDelimiter('#'));
        csvPrinter.printRecord("Name", "Price");
    }
    List<String> data = new ArrayList<String>();
    data.add(product.getName());
    data.add(String.valueOf(product.getPrice()));
    csvPrinter.printRecord(data);
    csvPrinter.flush();
// TODO
//        csvPrinter.close();
}

public void printToCSV(String testName) {
    checkStatisticsNotNull();
    for (String statistic : statistics.keySet()) {
        File csvFile = new File(PROJECT_BUILD_DIRECTORY + "/measurements" + (testName == null ? "" : "/" + testName), statistic + ".csv");
        boolean csvFileCreated = false;
        if (!csvFile.exists()) {
            try {
                csvFile.getParentFile().mkdirs();
                csvFileCreated = csvFile.createNewFile();
            } catch (IOException ex) {
                throw new RuntimeException(ex);
            }
        }
        try (BufferedWriter writer = new BufferedWriter(new FileWriter(csvFile, true))) {
            CSVPrinter printer = new CSVPrinter(writer, CSVFormat.RFC4180);
            if (csvFileCreated) {
                printer.printRecord(HEADER);
            }
            printer.printRecord(toRecord(statistic));
            printer.flush();
        } catch (IOException ex) {
            throw new RuntimeException(ex);
        }
    }
}

static CSVFormat createCSVFormat(final ConfigurationContext context) {
    final String formatName = context.getProperty(CSV_FORMAT).getValue();
    if (formatName.equalsIgnoreCase(CUSTOM.getValue())) {
        return buildCustomFormat(context);
    }
    if (formatName.equalsIgnoreCase(RFC_4180.getValue())) {
        return CSVFormat.RFC4180;
    } else if (formatName.equalsIgnoreCase(EXCEL.getValue())) {
        return CSVFormat.EXCEL;
    } else if (formatName.equalsIgnoreCase(TDF.getValue())) {
        return CSVFormat.TDF;
    } else if (formatName.equalsIgnoreCase(MYSQL.getValue())) {
        return CSVFormat.MYSQL;
    } else if (formatName.equalsIgnoreCase(INFORMIX_UNLOAD.getValue())) {
        return CSVFormat.INFORMIX_UNLOAD;
    } else if (formatName.equalsIgnoreCase(INFORMIX_UNLOAD_CSV.getValue())) {
        return CSVFormat.INFORMIX_UNLOAD_CSV;
    } else {
        return CSVFormat.DEFAULT;
    }
}

public static byte[] exportAsCSV(QueryResult result) {
    final ByteArrayOutputStream out = new ByteArrayOutputStream();
    final CSVPrinter csvPrinter;
    try {
        final CSVFormat format = CSVFormat.DEFAULT.withQuoteMode(QuoteMode.NON_NUMERIC);
        csvPrinter = new CSVPrinter(new PrintWriter(out), format);
        csvPrinter.printRecord(result.getMetadata().stream().map(SchemaField::getName).collect(Collectors.toList()));
        csvPrinter.printRecords(Iterables.transform(result.getResult(),  input -> Iterables.transform(input,  input1 -> {
            if (input1 instanceof List || input1 instanceof Map) {
                return JsonHelper.encode(input1);
            }
            if (input1 instanceof byte[]) {
                return DatatypeConverter.printBase64Binary((byte[]) input1);
            }
            return input1;
        })));
        csvPrinter.flush();
    } catch (IOException e) {
        throw Throwables.propagate(e);
    }
    return out.toByteArray();
}

private void printComponentsListAsCsv(List<ComponentInstallation> components, TextOutputReceiver outputReceiver, Map<LogicalNodeId, SystemLoadInformation> systemLoadData) {
    SortedSet<String> lines = new TreeSet<>();
    final CSVFormat csvFormat = CSVFormat.newFormat(' ').withQuote('"').withQuoteMode(QuoteMode.ALL);
    for (ComponentInstallation ci : components) {
        ComponentInterface compInterface = ci.getComponentRevision().getComponentInterface();
        String nodeId = ci.getNodeId();
        String nodeName = NodeIdentifierUtils.parseLogicalNodeIdStringWithExceptionWrapping(nodeId).getAssociatedDisplayName();
        if (systemLoadData != null) {
            final SystemLoadInformation loadDataEntry = systemLoadData.get(ci.fetchNodeIdAsObject());
            // two-step checking as there may be no load data available for that node
            final double cpuAvg;
            final int numSamples;
            final int timeSpan;
            final long availableRam;
            if (loadDataEntry != null) {
                final AverageOfDoubles cpuLoadAvg = loadDataEntry.getCpuLoadAvg();
                cpuAvg = cpuLoadAvg.getAverage() * PERCENT_MULTIPLIER;
                numSamples = cpuLoadAvg.getNumSamples();
                timeSpan = cpuLoadAvg.getNumSamples() * LocalSystemMonitoringAggregationService.SYSTEM_LOAD_INFORMATION_COLLECTION_INTERVAL_MSEC;
                availableRam = loadDataEntry.getAvailableRam();
            } else {
                cpuAvg = DOUBLE_NO_DATA_PLACEHOLDER;
                numSamples = INT_NO_DATA_PLACEHOLDER;
                timeSpan = INT_NO_DATA_PLACEHOLDER;
                availableRam = INT_NO_DATA_PLACEHOLDER;
            }
            lines.add(csvFormat.format(compInterface.getDisplayName(), compInterface.getVersion(), nodeId, nodeName, StringUtils.format("%.2f", cpuAvg), numSamples, timeSpan, availableRam));
        } else {
            lines.add(csvFormat.format(compInterface.getDisplayName(), compInterface.getVersion(), nodeId, nodeName));
        }
    }
    for (String line : lines) {
        outputReceiver.addOutput(line);
    }
}

@Override
public Object apply(final ActionContext ctx, final Object caller, final Object[] sources) {
    if (arrayHasMinLengthAndMaxLengthAndAllElementsNotNull(sources, 1, 4)) {
        try {
            final List<Map<String, String>> objects = new LinkedList<>();
            final String source = sources[0].toString();
            String delimiter = ";";
            String quoteChar = "\"";
            String recordSeparator = "\n";
            switch(sources.length) {
                case 4:
                    recordSeparator = (String) sources[3];
                case 3:
                    quoteChar = (String) sources[2];
                case 2:
                    delimiter = (String) sources[1];
                    break;
            }
            CSVFormat format = CSVFormat.newFormat(delimiter.charAt(0)).withHeader();
            format = format.withQuote(quoteChar.charAt(0));
            format = format.withRecordSeparator(recordSeparator);
            format = format.withIgnoreEmptyLines(true);
            format = format.withIgnoreSurroundingSpaces(true);
            format = format.withSkipHeaderRecord(true);
            format = format.withQuoteMode(QuoteMode.ALL);
            CSVParser parser = new CSVParser(new StringReader(source), format);
            for (final CSVRecord record : parser.getRecords()) {
                objects.add(record.toMap());
            }
            return objects;
        } catch (Throwable t) {
            logException(t, "{}: Exception for parameter: {}", new Object[] { getName(), getParametersAsString(sources) });
        }
        return "";
    } else {
        logParameterError(caller, sources, ctx.isJavaScriptContext());
    }
    return usage(ctx.isJavaScriptContext());
}

@Test
public void testZContextDependencyLoading() {
    // try to import library does not exist on classpath. it'll fail
    assertEquals(InterpreterResult.Code.ERROR, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
    // load library from maven repository and try to import again
    repl.interpret("z.load(\"org.apache.commons:commons-csv:1.1\")", context);
    assertEquals(InterpreterResult.Code.SUCCESS, repl.interpret("import org.apache.commons.csv.CSVFormat", context).code());
}

@Test
public void testSparkInterpreterDependencyLoading() throws Exception {
    if (!endToEndTestEnabled()) {
        return;
    }
    try {
        // navigate to interpreter page
        WebElement settingButton = driver.findElement(By.xpath("//button[@class='nav-btn dropdown-toggle ng-scope']"));
        settingButton.click();
        WebElement interpreterLink = driver.findElement(By.xpath("//a[@href='#/interpreter']"));
        interpreterLink.click();
        // add new dependency to spark interpreter
        driver.findElement(By.xpath("//div[@id='spark']//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
        WebElement depArtifact = pollingWait(By.xpath("//input[@ng-model='setting.depArtifact']"), MAX_BROWSER_TIMEOUT_SEC);
        String artifact = "org.apache.commons:commons-csv:1.1";
        depArtifact.sendKeys(artifact);
        driver.findElement(By.xpath("//div[@id='spark']//form//button[1]")).click();
        clickAndWait(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" + "//div[@class='modal-footer']//button[contains(.,'OK')]"));
        try {
            clickAndWait(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to " + "update this interpreter and restart with new settings?')]//" + "div[@class='bootstrap-dialog-close-button']/button"));
        } catch (TimeoutExceptionStaleElementReferenceException |  e) {
        }
        driver.navigate().back();
        createNewNote();
        // wait for first paragraph's " READY " status text
        waitForParagraph(1, "READY");
        setTextOfParagraph(1, "import org.apache.commons.csv.CSVFormat");
        runParagraph(1);
        waitForParagraph(1, "FINISHED");
        // check expected text
        WebElement paragraph1Result = driver.findElement(By.xpath(getParagraphXPath(1) + "//div[contains(@id,\"_text\")]"));
        collector.checkThat("Paragraph from ZeppelinIT of testSparkInterpreterDependencyLoading result: ", paragraph1Result.getText().toString(), CoreMatchers.containsString("import org.apache.commons.csv.CSVFormat"));
        //delete created notebook for cleanup.
        deleteTestNotebook(driver);
        ZeppelinITUtils.sleep(1000, false);
        // reset dependency
        settingButton.click();
        interpreterLink.click();
        driver.findElement(By.xpath("//div[@id='spark']//button[contains(.,'edit')]")).sendKeys(Keys.ENTER);
        WebElement testDepRemoveBtn = pollingWait(By.xpath("//tr[descendant::text()[contains(.,'" + artifact + "')]]/td[3]/button"), MAX_IMPLICIT_WAIT);
        testDepRemoveBtn.sendKeys(Keys.ENTER);
        driver.findElement(By.xpath("//div[@id='spark']//form//button[1]")).click();
        driver.findElement(By.xpath("//div[@class='modal-dialog'][contains(.,'Do you want to update this interpreter and restart with new settings?')]" + "//div[@class='modal-footer']//button[contains(.,'OK')]")).click();
    } catch (Exception e) {
        handleException("Exception in ZeppelinIT while testSparkInterpreterDependencyLoading ", e);
    }
}

public static List<KeyedMessage<String, String>> produceCsvMessages(String topic, String partition, CSVFormat csvFormat, File csvFile) throws IOException {
    List<KeyedMessage<String, String>> messages = new ArrayList<>();
    String line;
    BufferedReader bufferedReader = new BufferedReader(new FileReader(KafkaTestUtil.class.getClassLoader().getResource("testKafkaTarget.csv").getFile()));
    while ((line = bufferedReader.readLine()) != null) {
        String[] strings = line.split(",");
        StringWriter stringWriter = new StringWriter();
        CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat);
        csvPrinter.printRecord(strings);
        csvPrinter.flush();
        csvPrinter.close();
        messages.add(new KeyedMessage<>(topic, partition, stringWriter.toString()));
    }
    return messages;
}

public static Collection<Lead> importFile(File f) {
    try {
        Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(new FileReader(f));
        Iterator<CSVRecord> lines = records.iterator();
        if (!lines.hasNext())
            return null;
        CSVRecord headerRow = lines.next();
        List<Lead> leads = new ArrayList<>();
        while (lines.hasNext()) {
            final Iterator<String> content = lines.next().iterator();
            final Iterator<String> header = headerRow.iterator();
            Lead lead = new Lead(null, "new lead");
            StringBuilder desc = new StringBuilder();
            // use capterra as default campaign since it doesn't reference
            // itself in the import file
            Campaign camp = CrmManager.getCampaignByName("Capterra");
            if (camp != null)
                lead.setCampaignId(camp.getId());
            int column = 0;
            while (header.hasNext() && content.hasNext()) {
                column++;
                String key = header.next();
                String value = content.next();
                System.out.println(key + "=" + value);
                if (column == 1 && key.startsWith("RFQ")) {
                    camp = CrmManager.getCampaignByName("BuyerZone");
                    if (camp != null)
                        lead.setCampaignId(camp.getId());
                }
                switch(key) {
                    case "Campaign Code":
                        camp = CrmManager.getCampaignByName(value);
                        if (camp != null)
                            lead.setCampaignId(camp.getId());
                        break;
                    case "Organization":
                    case "Company":
                        lead.setAccountName(value);
                        break;
                    case "Seqment":
                        desc.append(value).append("\n\n");
                        break;
                    case "Size":
                        desc.append("Size: ").append(value).append("\n");
                        break;
                    case "Size 2":
                    case "# of Employees":
                        desc.append("Staff: ").append(value).append("\n");
                        break;
                    case "Size 3":
                        desc.append("Terminals: ").append(value).append("\n");
                        break;
                    case "Applications":
                        desc.append("Applications: ").append(value).append("\n");
                        break;
                    case "Deployment":
                        desc.append("Deployment: ").append(value).append("\n");
                        break;
                    case "Call Notes":
                    case "Problem Buyer Needs to Solve":
                        desc.append("\n\nCALL NOTES\n\n").append(value).append("\n\n");
                        break;
                    case "Budget Details":
                        desc.append("\n\nBudget Details\n\n").append(value).append("\n\n");
                    case "Buyer's Requirements":
                        desc.append("\n\nBudget Details\n\n").append(value).append("\n\n");
                        break;
                    case "Timeframe":
                    case "Purchase Timeframe":
                        desc.append("Timeframe: ").append(value).append("\n");
                        break;
                    case "Stage in Buying Process":
                        desc.append("Stage in Buying Process: ").append(value).append("\n");
                        break;
                    case "Product":
                    case "Type of Software Needed":
                        desc.append("Product: ").append(value).append("\n");
                        break;
                    case "Current Software":
                        desc.append("Current Software: ").append(value).append("\n");
                        break;
                    case "Buyer Has Budget":
                        desc.append("Buyer Has Budget: ").append(value).append("\n");
                        break;
                    case "Budget Amount":
                        desc.append("Budget Amount: ").append(value).append("\n");
                        break;
                    case "Decision Maker":
                        desc.append("Decision Maker: ").append(value).append("\n");
                        break;
                    case "Name":
                    case "Contact Name":
                        String[] split = value.split(" ");
                        if (split.length < 2)
                            lead.setLastName(value);
                        else {
                            lead.setFirstname(split[0]);
                            int i = 0;
                            StringBuilder sb = new StringBuilder();
                            for (String string : split) {
                                i++;
                                if (i == 1)
                                    continue;
                                sb.append(string);
                                if (i < split.length)
                                    sb.append(' ');
                            }
                            lead.setLastName(sb.toString());
                        }
                        break;
                    case "First Name":
                        lead.setFirstname(value);
                        break;
                    case "Last Name":
                        lead.setLastName(value);
                        break;
                    case "Job Title":
                    case "Title":
                        lead.setJobTitle(value);
                        break;
                    case "Phone":
                        lead.setPhone(value);
                        break;
                    case "Email":
                        lead.setEmail(value);
                        break;
                    case "Address":
                        lead.setAddress(value);
                        break;
                    case "City":
                        lead.setCity(value);
                        break;
                    case "State":
                        lead.setState(value);
                        break;
                    case "ZIP Code":
                    case "ZIP":
                    case "Zip":
                        lead.setZip(value);
                        break;
                    case "Country":
                    case "Location":
                        lead.setCountry(value);
                        break;
                    case "Request":
                        desc.append("Request: ").append(value).append("\n");
                        break;
                    case "Timestamp":
                        desc.append("Timestamp: ").append(value).append("\n");
                        break;
                    case "Appointment":
                        desc.append("\nAppointment: ").append(value).append("\n\n");
                        break;
                    default:
                        desc.append("").append(key).append(": ").append(value).append("\n\n");
                        break;
                }
            }
            lead.setDescription(desc.toString());
            lead.setType("Customer");
            leads.add(CrmManager.saveLead(lead));
        }
        return leads;
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}

private void exportData(Graph graph) throws Exception {
    final CSVFormat format = CSVFormat.DEFAULT.withDelimiter(fieldDelimiter);
    try (CSVPrinter csvWriter = new CSVPrinter(writer, format)) {
        boolean isEdgeTable = tableToExport != ExportTable.NODES;
        Table table = isEdgeTable ? graph.getModel().getEdgeTable() : graph.getModel().getNodeTable();
        ElementIterable<? extends Element> rows;
        Object[] edgeLabels = graph.getModel().getEdgeTypeLabels();
        boolean includeEdgeKindColumn = false;
        for (Object edgeLabel : edgeLabels) {
            if (edgeLabel != null && !edgeLabel.toString().isEmpty()) {
                includeEdgeKindColumn = true;
            }
        }
        TimeFormat timeFormat = graph.getModel().getTimeFormat();
        DateTimeZone timeZone = graph.getModel().getTimeZone();
        List<Column> columns = new ArrayList<>();
        if (columnIdsToExport != null) {
            for (String columnId : columnIdsToExport) {
                Column column = table.getColumn(columnId);
                if (column != null) {
                    columns.add(column);
                }
            }
        } else {
            for (Column column : table) {
                columns.add(column);
            }
        }
        //Write column headers:
        if (isEdgeTable) {
            csvWriter.print("Source");
            csvWriter.print("Target");
            csvWriter.print("Type");
            if (includeEdgeKindColumn) {
                csvWriter.print("Kind");
            }
        }
        for (Column column : columns) {
            //Use the title only if it's the same as the id (case insensitive):
            String columnId = column.getId();
            String columnTitle = column.getTitle();
            String columnHeader = columnId.equalsIgnoreCase(columnTitle) ? columnTitle : columnId;
            csvWriter.print(columnHeader);
        }
        csvWriter.println();
        //Write rows:
        if (isEdgeTable) {
            rows = graph.getEdges();
        } else {
            rows = graph.getNodes();
        }
        for (Element row : rows) {
            if (isEdgeTable) {
                Edge edge = (Edge) row;
                csvWriter.print(edge.getSource().getId());
                csvWriter.print(edge.getTarget().getId());
                csvWriter.print(edge.isDirected() ? "Directed" : "Undirected");
                if (includeEdgeKindColumn) {
                    csvWriter.print(edge.getTypeLabel().toString());
                }
            }
            for (Column column : columns) {
                Object value = row.getAttribute(column);
                String text;
                if (value != null) {
                    if (value instanceof Number) {
                        text = NUMBER_FORMAT.format(value);
                    } else {
                        text = AttributeUtils.print(value, timeFormat, timeZone);
                    }
                } else {
                    text = "";
                }
                csvWriter.print(text);
            }
            csvWriter.println();
        }
    }
}

private void reload() throws RemoteException {
    this.getError().setValue("");
    boolean enable = true;
    list.clear();
    header.clear();
    Integer headerline = ((Integer) getSkipLines().getValue());
    try {
        SWTUtil.disposeChildren(this.comp);
        comp.setLayoutData(new GridData(GridData.FILL_BOTH));
        this.comp.setLayout(new GridLayout());
        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName((String) getCharset().getValue())));
        String line = null;
        int counter = 0;
        while ((line = br.readLine()) != null && counter < 30) {
            GenericObjectHashMap g = new GenericObjectHashMap();
            g.setAttribute("Dateiinhalt", line);
            list.add(g);
            counter++;
        }
        header.add("Dateiinhalt");
        br.close();
        FileInputStream is = new FileInputStream(file);
        InputStreamReader isr = new InputStreamReader(is, Charset.forName((String) getCharset().getValue()));
        CSVFormat format = CSVFormat.RFC4180.withDelimiter(((String) getTrennzeichen().getValue()).charAt(0)).withSkipLines(headerline - 1);
        CSVParser parser = new CSVParser(isr, format);
        list.clear();
        boolean iserror = false;
        boolean isheader = true;
        header.clear();
        for (CSVRecord record : parser) {
            if (isheader) {
                for (int i = 0; i < record.size(); i++) {
                    String name = record.get(i);
                    if (name.isEmpty()) {
                        name = "Namenlos";
                    }
                    String neuername = name;
                    counter = 1;
                    while (header.contains(neuername)) {
                        neuername = name + " (" + counter + ")";
                        counter++;
                    }
                    header.add(neuername);
                }
                isheader = false;
                continue;
            }
            GenericObjectHashMap g = new GenericObjectHashMap();
            g.setAttribute("_DEPOTVIEWER_IGNORE", "");
            g.setAttribute("_DEPOTVIEWER_IDX", "" + (record.getRecordNumber() + headerline));
            for (int i = 0; i < header.size(); i++) {
                if (i >= record.size()) {
                    g.setAttribute("_DEPOTVIEWER_IGNORE", "X");
                    iserror = true;
                    continue;
                }
                g.setAttribute(header.get(i), record.get(i));
            }
            list.add(g);
        }
        if (iserror) {
            getError().setValue("Die mit X markierten Zeilen werden ignoriert.");
        }
        parser.close();
    } catch (Exception e) {
        Logger.error("unable to read file", e);
        this.getError().setValue("Fehler beim Lesen der Datei:\n" + e.getMessage());
        enable = false;
    }
    TablePart tab = new TablePart(list, null);
    tab.addColumn("" + headerline, "_DEPOTVIEWER_IDX");
    tab.addColumn("", "_DEPOTVIEWER_IGNORE");
    for (String h : header) {
        tab.addColumn(h, h);
    }
    tab.paint(comp);
    comp.layout(true);
    if (weiterbutton != null) {
        weiterbutton.setEnabled(enable);
    }
}

public static void assertResults(File outputFile, org.apache.samoa.TestParams testParams) throws IOException {
    LOG.info("Checking results file " + outputFile.getAbsolutePath());
    // 1. parse result file with csv parser
    Reader in = new FileReader(outputFile);
    Iterable<CSVRecord> records = CSVFormat.EXCEL.withSkipHeaderRecord(false).withIgnoreEmptyLines(true).withDelimiter(',').withCommentMarker('#').parse(in);
    CSVRecord last = null;
    Iterator<CSVRecord> iterator = records.iterator();
    CSVRecord header = iterator.next();
    // Results of Standard Evaluation have 5 columns, and cv Evaluation have 9 columns
    int cvEvaluation = (header.size() == 9) ? 1 : 0;
    String cvText = (header.size() == 9) ? "[avg] " : "";
    Assert.assertEquals("Unexpected column", org.apache.samoa.TestParams.EVALUATION_INSTANCES, header.get(0).trim());
    Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.CLASSIFIED_INSTANCES, header.get(1).trim());
    Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.CLASSIFICATIONS_CORRECT, header.get(2 + cvEvaluation).trim());
    Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.KAPPA_STAT, header.get(3 + 2 * cvEvaluation).trim());
    Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.KAPPA_TEMP_STAT, header.get(4 + 3 * cvEvaluation).trim());
    // 2. check last line result
    while (iterator.hasNext()) {
        last = iterator.next();
    }
    assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getEvaluationInstances(), Float.parseFloat(last.get(0))), testParams.getEvaluationInstances() <= Float.parseFloat(last.get(0)));
    assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getClassifiedInstances(), Float.parseFloat(last.get(1))), testParams.getClassifiedInstances() <= Float.parseFloat(last.get(1)));
    assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getClassificationsCorrect(), Float.parseFloat(last.get(2 + cvEvaluation))), testParams.getClassificationsCorrect() <= Float.parseFloat(last.get(2 + cvEvaluation)));
    assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaStat(), Float.parseFloat(last.get(3 + 2 * cvEvaluation))), testParams.getKappaStat() <= Float.parseFloat(last.get(3 + 2 * cvEvaluation)));
    assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaTempStat(), Float.parseFloat(last.get(4 + 3 * cvEvaluation))), testParams.getKappaTempStat() <= Float.parseFloat(last.get(4 + 3 * cvEvaluation)));
}

public static void main(String[] args) throws IOException {
    String inFolder = "/home/zqz/GDrive/papers/cicling2017/data/seed-terms/genia";
    String outFile = "/home/zqz/GDrive/papers/cicling2017/data/seed-terms/genia/voted.json";
    Map<String, Double> weights = new HashMap<>();
    weights.put("genia_attf_seed_terms.json", 1.0);
    weights.put("genia_chisquare_seed_terms.json", 1.0);
    weights.put("genia_cvalue_seed_terms.json", 1.0);
    weights.put("genia_cvalue_seed_terms_mttf1.json", 1.0);
    weights.put("genia_glossex_seed_terms.json", 1.0);
    weights.put("genia_rake_seed_terms.json", 1.0);
    weights.put("genia_termex_seed_terms.json", 1.0);
    weights.put("genia_tfidf_seed_terms.json", 1.0);
    weights.put("genia_ttf_seed_terms.json", 1.0);
    weights.put("genia_weirdness_seed_terms.json", 1.0);
    weights.put("genia_text_rank_result.csv", 1.0);
    Map<String, FileOutputReader> readers = new HashMap<>();
    FileOutputReader jsonFileOutputReader = new JSONFileOutputReader(new Gson());
    FileOutputReader csvFileOutputReader = new CSVFileOutputReader(CSVFormat.DEFAULT);
    readers.put("genia_attf_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_chisquare_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_cvalue_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_cvalue_seed_terms_mttf1.json", jsonFileOutputReader);
    readers.put("genia_glossex_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_rake_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_termex_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_tfidf_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_ttf_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_weirdness_seed_terms.json", jsonFileOutputReader);
    readers.put("genia_text_rank_result.csv", csvFileOutputReader);
    Voting voting = new Voting();
    Pair[] results = voting.readAlgorithmResults(inFolder, weights, readers);
    List<JATETerm> newResult = voting.vote(results);
    Writer w = IOUtil.getUTF8Writer(outFile);
    new Gson().toJson(newResult, w);
    w.close();
}

/**
     * Evaluates the given transformation.
     *
     * This first parses the document (with the first line being the header) and then evaluates on our framework.
     *
     * TODO: this assumes a header line. It might happen that it is not provided.
     *
     * @param transformation
     * @return
     */
@Override
public JavaRDD<ObjectValue> evaluate(Transformation transformation) {
    final SparkTransformationEvaluator evaluator = new SparkTransformationEvaluator(transformation);
    final JavaSparkContext sc = NotaQL.SparkFactory.getSparkContext();
    final CSVFormat format = CSVFormat.DEFAULT;
    final JavaRDD<String> csv = sc.textFile(path);
    final String first = csv.first();
    final CSVRecord header;
    try {
        header = format.parse(new StringReader(first)).iterator().next();
    } catch (IOException e) {
        e.printStackTrace();
        throw new AssertionError("Header could not be read for some reason.");
    }
    String[] headerCols = new String[header.size()];
    for (int i = 0; i < header.size(); i++) {
        headerCols[i] = header.get(i);
    }
    final CSVFormat headerFormat = CSVFormat.DEFAULT.withHeader(headerCols);
    final JavaRDD<CSVRecord> records = csv.filter( f -> !f.equals(first)).map( line -> headerFormat.parse(new StringReader(line)).iterator().next());
    final JavaRDD<Value> converted = records.map(ValueConverter::convertToNotaQL);
    final JavaRDD<Value> filtered = converted.filter( o -> transformation.satisfiesInPredicate((ObjectValue) o));
    return evaluator.process(filtered);
}

public void doImport(CoreSession session) {
    UserProfileService ups = Framework.getLocalService(UserProfileService.class);
    config = ups.getImporterConfig();
    if (config == null) {
        log.error("No importer configuration could be found");
        return;
    }
    dataFileName = config.getDataFileName();
    if (dataFileName == null) {
        log.error("No importer dataFileName was supplied");
        return;
    }
    InputStream is = getResourceAsStream(dataFileName);
    if (is == null) {
        log.error("Error locating CSV data file: " + dataFileName);
        return;
    }
    Reader in = new BufferedReader(new InputStreamReader(is));
    CSVParser parser = null;
    try {
        parser = CSVFormat.DEFAULT.withEscape(escapeCharacter).withHeader().parse(in);
        doImport(session, parser, ups);
    } catch (IOException e) {
        log.error("Unable to read CSV file", e);
    } finally {
        if (parser != null) {
            try {
                parser.close();
            } catch (IOException e) {
                log.debug(e, e);
            }
        }
    }
}

protected void parseCSV(Reader in, String[] header, PerformanceReport report) throws IOException {
    CSVFormat csvFormat = CSVFormat.newFormat(delimiter).withHeader(header).withQuote('"').withSkipHeaderRecord();
    Iterable<CSVRecord> records = csvFormat.parse(in);
    for (CSVRecord record : records) {
        final HttpSample sample = getSample(record);
        try {
            report.addSample(sample);
        } catch (SAXException e) {
            throw new RuntimeException("Error parsing file '" + report.getReportFileName() + "': Unable to add sample for CSVRecord " + record, e);
        }
    }
}

/* **************************************************************************
   * DataFrame parsers
   ***************************************************************************/
/**
   * Reads in a CSV structured stream and returns it as a DataFrame. The native series type is
   * chosen to be as specific as possible based on the data ingested.
   * <br/><b>NOTE:</b> Expects the first line to contain
   * column headers. The column headers are transformed into series names by replacing non-word
   * character sequences with underscores ({@code "_"}). Leading digits in series names are also
   * escaped with a leading underscore.
   *
   * @param in input reader
   * @return CSV as DataFrame
   * @throws IOException if a read error is encountered
   * @throws IllegalArgumentException if the column headers cannot be transformed into valid series names
   */
public static DataFrame fromCsv(Reader in) throws IOException {
    Iterator<CSVRecord> it = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in).iterator();
    if (!it.hasNext())
        return new DataFrame();
    CSVRecord first = it.next();
    Set<String> headers = first.toMap().keySet();
    // transform column headers into series names
    Map<String, String> header2name = new HashMap<>();
    for (String h : headers) {
        // remove spaces
        String name = Pattern.compile("\\W+").matcher(h).replaceAll("_");
        // underscore escape leading number
        if (Pattern.compile("\\A[0-9]").matcher(name).find())
            name = "_" + name;
        if (!SERIES_NAME_PATTERN.matcher(name).matches()) {
            throw new IllegalArgumentException(String.format("Series name must match pattern '%s'", SERIES_NAME_PATTERN));
        }
        header2name.put(h, name);
    }
    // read first line and initialize builders
    Map<String, StringSeries.Builder> builders = new HashMap<>();
    for (String h : headers) {
        StringSeries.Builder builder = StringSeries.builder();
        builder.addValues(first.get(h));
        builders.put(h, builder);
    }
    while (it.hasNext()) {
        CSVRecord record = it.next();
        for (String h : headers) {
            String value = record.get(h);
            builders.get(h).addValues(value);
        }
    }
    // construct dataframe and detect native data types
    DataFrame df = new DataFrame();
    for (Map.Entry<String, StringSeries.Builder> e : builders.entrySet()) {
        StringSeries s = e.getValue().build();
        Series conv = s.get(s.inferType());
        String name = header2name.get(e.getKey());
        df.addSeries(name, conv);
    }
    return df;
}

public static void assertResults(File outputFile, com.yahoo.labs.samoa.TestParams testParams) throws IOException {
    LOG.info("Checking results file " + outputFile.getAbsolutePath());
    // 1. parse result file with csv parser
    Reader in = new FileReader(outputFile);
    Iterable<CSVRecord> records = CSVFormat.EXCEL.withSkipHeaderRecord(false).withIgnoreEmptyLines(true).withDelimiter(',').withCommentMarker('#').parse(in);
    CSVRecord last = null;
    Iterator<CSVRecord> iterator = records.iterator();
    CSVRecord header = iterator.next();
    Assert.assertEquals("Invalid number of columns", 5, header.size());
    Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.EVALUATION_INSTANCES, header.get(0).trim());
    Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.CLASSIFIED_INSTANCES, header.get(1).trim());
    Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.CLASSIFICATIONS_CORRECT, header.get(2).trim());
    Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.KAPPA_STAT, header.get(3).trim());
    Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.KAPPA_TEMP_STAT, header.get(4).trim());
    // 2. check last line result
    while (iterator.hasNext()) {
        last = iterator.next();
    }
    assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getEvaluationInstances(), Float.parseFloat(last.get(0))), testParams.getEvaluationInstances() <= Float.parseFloat(last.get(0)));
    assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getClassifiedInstances(), Float.parseFloat(last.get(1))), testParams.getClassifiedInstances() <= Float.parseFloat(last.get(1)));
    assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getClassificationsCorrect(), Float.parseFloat(last.get(2))), testParams.getClassificationsCorrect() <= Float.parseFloat(last.get(2)));
    assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaStat(), Float.parseFloat(last.get(3))), testParams.getKappaStat() <= Float.parseFloat(last.get(3)));
    assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaTempStat(), Float.parseFloat(last.get(4))), testParams.getKappaTempStat() <= Float.parseFloat(last.get(4)));
}

/* (non-Javadoc)
	 * @see com.brightcove.opensource.workflowengine.Actor#run()
	 */
public void run() {
    Property<String> inputFileProp = getFirstProperty("input-file");
    if ((inputFileProp == null) || (inputFileProp.getValue() == null)) {
        die("CSVAdapter requires an input-file property.");
    }
    File inputFile = new File(inputFileProp.getValue());
    if (!inputFile.exists()) {
        die("[ERR] CSVAdapter input file (" + inputFile.getAbsolutePath() + ") does not exist.");
    }
    Character delimiter = ',';
    Character quote = '"';
    String delimiterProp = getFirstPropertyValue("delimiter");
    if (delimiterProp != null) {
        delimiter = delimiterProp.charAt(0);
    }
    String quoteProp = getFirstPropertyValue("quote");
    if (quoteProp != null) {
        quote = quoteProp.charAt(0);
    }
    CSVRecord headerRow = null;
    String headerRowProp = getFirstPropertyValue("header-row");
    if (headerRowProp != null) {
        Reader reader = new StringReader(headerRowProp);
        ;
        Iterable<CSVRecord> parser = null;
        try {
            parser = CSVFormat.newBuilder().withDelimiter(delimiter).withQuoteChar(quote).parse(reader);
        } catch (Exception e) {
            die("[ERR] CSVAdapter can't parse header row (" + headerRowProp + ").  Exception caught: '" + e + "'.");
        }
        for (CSVRecord csvRecord : parser) {
            if (headerRow == null) {
                headerRow = csvRecord;
            }
        }
    }
    Boolean hasHeaderRow = false;
    String hasHeaderRowProp = getFirstPropertyValue("has-header-row");
    if ("true".equalsIgnoreCase(hasHeaderRowProp)) {
        hasHeaderRow = true;
    }
    if (!hasStarted()) {
        FileReader reader = null;
        Iterable<CSVRecord> parser = null;
        try {
            reader = new FileReader(inputFile);
            parser = CSVFormat.newBuilder().withDelimiter(delimiter).withQuoteChar(quote).parse(reader);
        } catch (Exception e) {
            die("[ERR] CSVAdapter can't parse input file (" + inputFile.getAbsolutePath() + ").  Exception caught: '" + e + "'.");
        }
        Integer rowIdx = 0;
        for (CSVRecord csvRecord : parser) {
            rowIdx++;
            if (hasHeaderRow && (rowIdx == 1)) {
                if (headerRow == null) {
                    // Only use header row from file if it wasn't supplied by property
                    headerRow = csvRecord;
                }
            } else {
                Record record = new Record();
                for (Integer idx = 0; idx < csvRecord.size(); idx++) {
                    String propName = "Column-" + idx;
                    if ((headerRow != null) && (idx < headerRow.size())) {
                        propName = headerRow.get(idx);
                    }
                    Property<?> prop = new Property<String>(propName, csvRecord.get(idx));
                    record.addProperty(prop);
                }
                handleRecord(record);
            }
        }
    }
}

private void exportData(Graph graph) throws Exception {
    final CSVFormat format = CSVFormat.DEFAULT.withDelimiter(fieldDelimiter);
    try (CSVPrinter csvWriter = new CSVPrinter(writer, format)) {
        boolean isEdgeTable = tableToExport != ExportTable.NODES;
        Table table = isEdgeTable ? graph.getModel().getEdgeTable() : graph.getModel().getNodeTable();
        ElementIterable<? extends Element> rows;
        Object[] edgeLabels = graph.getModel().getEdgeTypeLabels();
        boolean includeEdgeKindColumn = false;
        for (Object edgeLabel : edgeLabels) {
            if (edgeLabel != null && !edgeLabel.toString().isEmpty()) {
                includeEdgeKindColumn = true;
            }
        }
        TimeFormat timeFormat = graph.getModel().getTimeFormat();
        DateTimeZone timeZone = graph.getModel().getTimeZone();
        List<Column> columns = new ArrayList<>();
        if (columnIdsToExport != null) {
            for (String columnId : columnIdsToExport) {
                Column column = table.getColumn(columnId);
                if (column != null) {
                    columns.add(column);
                }
            }
        } else {
            for (Column column : table) {
                columns.add(column);
            }
        }
        //Write column headers:
        if (isEdgeTable) {
            csvWriter.print("Source");
            csvWriter.print("Target");
            csvWriter.print("Type");
            if (includeEdgeKindColumn) {
                csvWriter.print("Kind");
            }
        }
        for (Column column : columns) {
            //Use the title only if it's the same as the id (case insensitive):
            String columnId = column.getId();
            String columnTitle = column.getTitle();
            String columnHeader = columnId.equalsIgnoreCase(columnTitle) ? columnTitle : columnId;
            csvWriter.print(columnHeader);
        }
        csvWriter.println();
        //Write rows:
        if (isEdgeTable) {
            rows = graph.getEdges();
        } else {
            rows = graph.getNodes();
        }
        for (Element row : rows) {
            if (isEdgeTable) {
                Edge edge = (Edge) row;
                csvWriter.print(edge.getSource().getId());
                csvWriter.print(edge.getTarget().getId());
                csvWriter.print(edge.isDirected() ? "Directed" : "Undirected");
                if (includeEdgeKindColumn) {
                    csvWriter.print(edge.getTypeLabel().toString());
                }
            }
            for (Column column : columns) {
                Object value = row.getAttribute(column);
                String text;
                if (value != null) {
                    if (value instanceof Number) {
                        text = NUMBER_FORMAT.format(value);
                    } else {
                        text = AttributeUtils.print(value, timeFormat, timeZone);
                    }
                } else {
                    text = "";
                }
                csvWriter.print(text);
            }
            csvWriter.println();
        }
    }
}

public ConcurrentMap<String, Partner> getTepunaPartners() {
    String prefix = "NLNZ";
    String institutionCode = config.getProperty("ladd.institution.code");
    ConcurrentMap<String, Partner> result = new ConcurrentHashMap<String, Partner>();
    WebClient webClient = webClientProvider.get();
    TextPage page = null;
    try {
        log.debug("tepuna url: {}", tepunaUrl);
        page = webClient.getPage(tepunaUrl);
    } catch (IOException e) {
        log.error("unable to acquire page: {}", tepunaUrl);
        return result;
    }
    log.debug("{}", page.getContent());
    try (CSVParser parser = CSVParser.parse(page.getContent(), CSVFormat.DEFAULT.withHeader())) {
        for (CSVRecord record : parser) {
            String nuc = record.get(0);
            if ("NUC symbol".equals(nuc) || institutionCode.equals(nuc)) {
                log.debug("skipping nuc: {}", nuc);
                continue;
            }
            nuc = prefix + ":" + nuc;
            String org = record.get(2);
            Partner partner = new Partner();
            partner.setLink("https://api-ap.hosted.exlibrisgroup.com/almaws/v1/partners/" + nuc);
            PartnerDetails partnerDetails = new PartnerDetails();
            partner.setPartnerDetails(partnerDetails);
            ProfileDetails profileDetails = new ProfileDetails();
            partnerDetails.setProfileDetails(profileDetails);
            profileDetails.setProfileType(ProfileType.ISO);
            RequestExpiryType requestExpiryType = new RequestExpiryType();
            requestExpiryType.setValue("INTEREST_DATE");
            requestExpiryType.setDesc("Expire by interest date");
            IsoDetails isoDetails = new IsoDetails();
            profileDetails.setIsoDetails(isoDetails);
            isoDetails.setAlternativeDocumentDelivery(false);
            isoDetails.setIllServer(config.getProperty("alma.ill.server"));
            isoDetails.setIllPort(Integer.parseInt(config.getProperty("alma.ill.port")));
            isoDetails.setIsoSymbol(nuc);
            isoDetails.setSendRequesterInformation(false);
            isoDetails.setSharedBarcodes(true);
            isoDetails.setRequestExpiryType(requestExpiryType);
            SystemType systemType = new SystemType();
            systemType.setValue("LADD");
            systemType.setDesc("LADD");
            LocateProfile locateProfile = new LocateProfile();
            locateProfile.setValue("LADD");
            locateProfile.setDesc("LADD Locate Profile");
            partnerDetails.setStatus(Status.ACTIVE);
            partnerDetails.setCode(nuc);
            partnerDetails.setName(org);
            partnerDetails.setSystemType(systemType);
            partnerDetails.setAvgSupplyTime(4);
            partnerDetails.setDeliveryDelay(4);
            partnerDetails.setCurrency("AUD");
            partnerDetails.setBorrowingSupported(true);
            partnerDetails.setBorrowingWorkflow("LADD_Borrowing");
            partnerDetails.setLendingSupported(true);
            partnerDetails.setLendingWorkflow("LADD_Lending");
            partnerDetails.setLocateProfile(locateProfile);
            partnerDetails.setHoldingCode(nuc);
            ContactInfo contactInfo = new ContactInfo();
            partner.setContactInfo(contactInfo);
            Addresses addresses = new Addresses();
            contactInfo.setAddresses(addresses);
            String s = record.get(5);
            if (s == null || "".equals(s.trim()))
                s = record.get(4);
            if (s != null && !"".equals(s.trim())) {
                Address address = getAddress(s);
                address.setPreferred(true);
                address.setAddressTypes(new AddressTypes());
                address.getAddressTypes().getAddressType().add("ALL");
                addresses.getAddress().add(address);
                log.debug("nuc/address [{}]: {}", nuc, address);
            }
            Emails emails = new Emails();
            contactInfo.setEmails(emails);
            s = record.get(6);
            if (s != null && !"".equals(s.trim())) {
                Email email = new Email();
                email.setEmailTypes(new EmailTypes());
                email.setEmailAddress(s);
                email.setPreferred(true);
                email.setDescription("Primary Email Address");
                email.getEmailTypes().getEmailType().add("ALL");
                emails.getEmail().add(email);
                log.debug("nuc/email1 [{}]: {}", nuc, email);
            }
            s = record.get(13);
            if (s != null && !"".equals(s.trim())) {
                Email email = new Email();
                email.setEmailTypes(new EmailTypes());
                email.setEmailAddress(s);
                email.setPreferred(true);
                String m = record.get(12);
                if (m != null && !"".equals(m))
                    email.setDescription("Manager Email Address: " + m);
                else
                    email.setDescription("Manager Email Address");
                email.getEmailTypes().getEmailType().add("ALL");
                emails.getEmail().add(email);
                log.debug("nuc/email2 [{}]: {}", nuc, email);
            }
            Phones phones = new Phones();
            contactInfo.setPhones(phones);
            s = record.get(15);
            if (s == null || "".equals(s.trim()))
                s = record.get(7);
            if (s != null && !"".equals(s.trim())) {
                Phone phone = new Phone();
                phone.setPhoneTypes(new PhoneTypes());
                phone.setPhoneNumber(s);
                phone.setPreferred(true);
                phone.setPreferredSMS(false);
                phone.getPhoneTypes().getPhoneType().add("ALL");
                phones.getPhone().add(phone);
                log.debug("nuc/phone [{}]: {}", nuc, phone);
            }
            Notes notes = new Notes();
            partner.setNotes(notes);
            result.put(partner.getPartnerDetails().getCode(), partner);
        }
    } catch (IOException ioe) {
        log.error("unable to parse data: {}", tepunaUrl);
    }
    return result;
}

/**
     * Load the BIN range information from disk. If the map has already been
     * initialized, don't load again.
     */
private static synchronized void loadBINRanges() {
    if (binsLoaded == false) {
        try {
            //NON-NLS
            InputStreamReader in = new InputStreamReader(CreditCards.class.getResourceAsStream("ranges.csv"));
            CSVParser rangesParser = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in);
            //parse each row and add to range map
            for (CSVRecord record : rangesParser) {
                /**
                     * Because ranges.csv allows both 6 and (the newer) 8 digit
                     * BINs, but we need a consistent length for the range map,
                     * we pad all the numbers out to 8 digits
                     */
                //pad start with 0's //NON-NLS
                String start = StringUtils.rightPad(record.get("iin_start"), 8, "0");
                //if there is no end listed, use start, since ranges will be closed.
                //NON-NLS
                String end = StringUtils.defaultIfBlank(record.get("iin_end"), start);
                //pad end with 9's //NON-NLS
                end = StringUtils.rightPad(end, 8, "99");
                //NON-NLS
                final String numberLength = record.get("number_length");
                try {
                    BINRange binRange = new BINRange(Integer.parseInt(start), Integer.parseInt(end), StringUtils.isBlank(numberLength) ? null : Integer.valueOf(numberLength), //NON-NLS
                    record.get("scheme"), //NON-NLS
                    record.get("brand"), //NON-NLS
                    record.get("type"), //NON-NLS
                    record.get("country"), //NON-NLS
                    record.get("bank_name"), //NON-NLS
                    record.get("bank_url"), //NON-NLS
                    record.get("bank_phone"), //NON-NLS
                    record.get("bank_city"));
                    binRanges.put(Range.closed(binRange.getBINstart(), binRange.getBINend()), binRange);
                } catch (NumberFormatException numberFormatException) {
                    LOGGER.log(Level.WARNING, "Failed to parse BIN range: " + record.toString(), numberFormatException);
                }
                binsLoaded = true;
            }
        } catch (IOException ex) {
            LOGGER.log(Level.WARNING, "Failed to load BIN ranges form ranges.csv", ex);
            MessageNotifyUtil.Notify.warn("Credit Card Number Discovery", "There was an error loading Bank Identification Number information.  Accounts will not have their BINs identified.");
        }
    }
}

//Validation/Conversion Layer function
public void validateAndNormalizeInputData(S3Event event, Context ctx) throws Exception {
    AmazonS3 s3Client;
    InputStream inputFileStream = null;
    InputStream readableDataStream = null;
    List<S3EventNotificationRecord> notificationRecords = event.getRecords();
    s3Client = new AmazonS3Client();
    String eventFileName, siteName, dbfName;
    CSVParser fileParser = null;
    for (S3EventNotificationRecord record : notificationRecords) {
        eventFileName = record.getS3().getObject().getKey();
        S3Object s3Object = s3Client.getObject(new GetObjectRequest(record.getS3().getBucket().getName(), record.getS3().getObject().getKey()));
        inputFileStream = s3Object.getObjectContent();
        fileParser = new CSVParser(new InputStreamReader(inputFileStream), CSVFormat.TDF.withCommentMarker('-'));
        List<CSVRecord> records = fileParser.getRecords();
        StringWriter writer = new StringWriter();
        CSVPrinter printer = null;
        if (records.get(0).toString().matches(".*[^0-9].*")) {
            records.remove(0);
        }
        printer = new CSVPrinter(writer, CSVFormat.DEFAULT.withRecordSeparator(System.getProperty("line.separator")));
        printer.printRecords(records);
        printer.flush();
        readableDataStream = new ByteArrayInputStream(writer.toString().getBytes("utf-8"));
        s3Client.putObject(record.getS3().getBucket().getName(), "validated/" + eventFileName + ".csv", readableDataStream, new ObjectMetadata());
        printer.close();
        readableDataStream.close();
    }
}

@RequestMapping(value = "{pid}", method = RequestMethod.GET)
public void export(@PathVariable("pid") String pid, HttpServletRequest request, HttpServletResponse response) throws IOException {
    String filename = pid.replace(":", "_") + ".csv";
    response.addHeader("Content-Disposition", "attachment; filename=\"" + filename + "\"");
    response.addHeader("Content-Type", "text/csv");
    SearchRequest searchRequest = generateSearchRequest(request, searchStateFactory.createSearchState());
    SearchState searchState = searchRequest.getSearchState();
    searchState.setResultFields(Arrays.asList(SearchFieldKeys.ID.name(), SearchFieldKeys.TITLE.name(), SearchFieldKeys.RESOURCE_TYPE.name(), SearchFieldKeys.ANCESTOR_IDS.name(), SearchFieldKeys.STATUS.name(), SearchFieldKeys.DATASTREAM.name(), SearchFieldKeys.ANCESTOR_PATH.name(), SearchFieldKeys.CONTENT_MODEL.name(), SearchFieldKeys.DATE_ADDED.name(), SearchFieldKeys.DATE_UPDATED.name(), SearchFieldKeys.LABEL.name(), SearchFieldKeys.CONTENT_STATUS.name()));
    searchState.setSortType("export");
    searchState.setRowsPerPage(searchSettings.maxPerPage);
    BriefObjectMetadata container = queryLayer.addSelectedContainer(pid, searchState, false);
    SearchResultResponse resultResponse = queryLayer.getSearchResults(searchRequest);
    List<BriefObjectMetadata> objects = resultResponse.getResultList();
    objects.add(0, container);
    queryLayer.getChildrenCounts(objects, searchRequest);
    try (ServletOutputStream out = response.getOutputStream()) {
        Writer writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
        try (CSVPrinter printer = CSVFormat.EXCEL.print(writer)) {
            printHeaders(printer);
            for (BriefObjectMetadata object : objects) {
                printObject(printer, object);
            }
        }
    }
}

private CSVFormat getCSVFormat() {
    CSVFormat format = CSVFormat.DEFAULT.withEscape('\\');
    if (myCsvOptions.sSeparatedChar.length() == 1) {
        format = format.withDelimiter(myCsvOptions.sSeparatedChar.charAt(0));
    }
    if (myCsvOptions.sSeparatedTextChar.length() == 1) {
        format = format.withQuote(myCsvOptions.sSeparatedTextChar.charAt(0));
    }
    return format;
}

/**
	 * Print result of transformation matrix to csv file.
	*/
public void printResultHorizontal(ArrayList<Long> timestamps, ArrayList<ArrayList<Double>> matrix, ArrayList<String> features, String directoryName, String fileName) {
    ByteArrayOutputStream bout = new ByteArrayOutputStream();
    OutputStreamWriter osw = new OutputStreamWriter(bout);
    try {
        CSVPrinter csv = new CSVPrinter(osw, CSVFormat.DEFAULT);
        //print the header of the matrix
        for (String f : features) {
            csv.print(f);
        }
        csv.println();
        //iterate per each row of the matrix
        for (int i = 0; i < matrix.size(); i++) {
            if (timestamps != null)
                csv.print(timestamps.get(0));
            for (int j = 0; j < matrix.get(i).size(); j++) {
                csv.print(matrix.get(i).get(j));
            }
            csv.println();
        }
        csv.flush();
        csv.close();
    } catch (IOException e1) {
        e1.printStackTrace();
    }
    OutputStream outputStream;
    try {
        //create the output directory if it doesn't exist
        File dir = new File(directoryName);
        dir.mkdirs();
        outputStream = new FileOutputStream(dir.getPath().concat(fileName));
        bout.writeTo(outputStream);
        bout.close();
        outputStream.close();
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
}

private CSVFormat createCSVFormat(String sampleData) throws IOException {
    CSVFormat format;
    if (autoDetect) {
        CSVAutoDetect autoDetect = new CSVAutoDetect();
        format = autoDetect.detectCSVFormat(sampleData, this.headerRow);
        this.separatorChar = Character.toString(format.getDelimiter());
        this.quoteChar = Character.toString(format.getQuoteCharacter());
    } else {
        format = CSVFormat.DEFAULT.withAllowMissingColumnNames();
        if (StringUtils.isNotEmpty(separatorChar)) {
            format = format.withDelimiter(toChar(separatorChar).charAt(0));
        }
        if (StringUtils.isNotEmpty(escapeChar)) {
            format = format.withEscape(toChar(escapeChar).charAt(0));
        }
        if (StringUtils.isNotEmpty(quoteChar)) {
            format = format.withQuoteMode(QuoteMode.MINIMAL).withQuote(toChar(quoteChar).charAt(0));
        }
    }
    return format;
}

public void parseInputCSV() throws IOException {
    final Reader reader = new InputStreamReader(inputFile.toURL().openStream(), "UTF-8");
    Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(reader);
    for (CSVRecord record : records) {
        String key_name = record.get(KEY_COLUMN);
        String en_value = record.get(EN_COLUMN);
        String nl_value = record.get(NL_COLUMN);
        String de_value = record.get(DE_COLUMN);
        System.out.println(key_name);
        System.out.println(en_value);
        System.out.println(nl_value);
        System.out.println(de_value);
        translationsEN.put(key_name, en_value);
        translationsNL.put(key_name, nl_value);
        translationsDE.put(key_name, de_value);
    }
}

public void processCsvStream(InputStream rawIn, Mapping mapping, GraphPropertyWorkData data) throws IOException {
    Reader reader = new InputStreamReader(rawIn);
    Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(reader);
    State state = new State(mapping, data);
    for (CSVRecord record : records) {
        if (record.getRecordNumber() <= mapping.getLinesToSkip()) {
            continue;
        }
        state.setRecord(record);
        processCsvLine(state);
    }
    getGraph().flush();
}

public List<CSVRecord> siteLocationData(int siteIndex) {
    //return cached results if we have them
    //TODO invalidate cache when needed (after a day?)
    List<CSVRecord> cachedLocations = locationsBySite.get(siteIndex);
    if (cachedLocations != null)
        return cachedLocations;
    List<CSVRecord> locations = new ArrayList<>();
    URL locationsURL = null;
    try {
        String sixDigitSite = String.format("%06d", siteIndex);
        locationsURL = new URL(getURLPrefix() + "/locations/site_" + sixDigitSite + ".csv");
        Log.d(TAG, "location url: %@" + locationsURL);
        HttpURLConnection urlConnection = (HttpURLConnection) locationsURL.openConnection();
        try {
            InputStream in = new BufferedInputStream(urlConnection.getInputStream());
            final Reader reader = new InputStreamReader(in);
            final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT);
            try {
                for (final CSVRecord record : parser) {
                    locations.add(record);
                }
                locationsBySite.put(siteIndex, locations);
            } finally {
                parser.close();
                reader.close();
            }
        } finally {
            urlConnection.disconnect();
        }
    } catch (MalformedURLException e) {
        Log.e(TAG, "Badly formatted URL for location manifest: " + locationsURL);
    } catch (IOException e) {
        Log.e(TAG, "Error reading from location manifest URL: " + locationsURL);
    }
    return locations;
}

@Override
protected void init(final ImmutableMap<String, Object> execEnvVars) throws DbException {
    buffer = new TupleBatchBuffer(getSchema());
    try {
        parser = new CSVParser(new BufferedReader(new InputStreamReader(source.getInputStream())), CSVFormat.newFormat(delimiter).withQuote(quote).withEscape(escape));
        iterator = parser.iterator();
        for (int i = 0; i < numberOfSkippedLines; i++) {
            iterator.next();
        }
    } catch (IOException e) {
        throw new DbException(e);
    }
    lineNumber = 0;
}

private void loadData() throws DirectoryException {
    log.debug("loading data file: " + dataFileName);
    CSVParser csvParser = null;
    PreparedStatement ps = null;
    try {
        InputStream is = getClass().getClassLoader().getResourceAsStream(dataFileName);
        if (is == null) {
            is = Framework.getResourceLoader().getResourceAsStream(dataFileName);
            if (is == null) {
                throw new DirectoryException("data file not found: " + dataFileName);
            }
        }
        csvParser = new CSVParser(new InputStreamReader(is, SQL_SCRIPT_CHARSET), CSVFormat.DEFAULT.withDelimiter(characterSeparator).withHeader());
        Map<String, Integer> header = csvParser.getHeaderMap();
        List<Column> columns = new ArrayList<>();
        Insert insert = new Insert(table);
        for (String columnName : header.keySet()) {
            String trimmedColumnName = columnName.trim();
            Column column = table.getColumn(trimmedColumnName);
            if (column == null) {
                throw new DirectoryException("column not found: " + trimmedColumnName);
            }
            columns.add(table.getColumn(trimmedColumnName));
            insert.addColumn(column);
        }
        String insertSql = insert.getStatement();
        log.debug("insert statement: " + insertSql);
        ps = connection.prepareStatement(insertSql);
        for (CSVRecord record : csvParser) {
            if (record.size() == 0 || record.size() == 1 && StringUtils.isBlank(record.get(0))) {
                // empty lines
                continue;
            }
            if (!record.isConsistent()) {
                log.error("invalid column count while reading CSV file: " + dataFileName + ", values: " + record);
                continue;
            }
            if (logger.isLogEnabled()) {
                List<Serializable> values = new ArrayList<>(header.size());
                for (String value : record) {
                    if (SQL_NULL_MARKER.equals(value)) {
                        value = null;
                    }
                    values.add(value);
                }
                logger.logSQL(insertSql, values);
            }
            for (int i = 0; i < header.size(); i++) {
                Column column = columns.get(i);
                String value = record.get(i);
                Serializable v;
                try {
                    if (SQL_NULL_MARKER.equals(value)) {
                        v = null;
                    } else if (column.getType().spec == ColumnSpec.STRING) {
                        v = value;
                    } else if (column.getType().spec == ColumnSpec.BOOLEAN) {
                        v = Boolean.valueOf(value);
                    } else if (column.getType().spec == ColumnSpec.LONG) {
                        v = Long.valueOf(value);
                    } else if (column.getType().spec == ColumnSpec.TIMESTAMP) {
                        Calendar cal = new GregorianCalendar();
                        cal.setTime(Timestamp.valueOf(value));
                        v = cal;
                    } else if (column.getType().spec == ColumnSpec.DOUBLE) {
                        v = Double.valueOf(value);
                    } else {
                        throw new DirectoryException("unrecognized column type: " + column.getType() + ", values: " + record);
                    }
                    column.setToPreparedStatement(ps, i + 1, v);
                } catch (IllegalArgumentException e) {
                    throw new DirectoryException(String.format("failed to set column '%s' on table '%s', values: %s", column.getPhysicalName(), table.getPhysicalName(), record), e);
                } catch (SQLException e) {
                    throw new DirectoryException(String.format("Table '%s' initialization failed: %s, values: %s", table.getPhysicalName(), e.getMessage(), record), e);
                }
            }
            ps.execute();
        }
    } catch (IOException e) {
        throw new DirectoryException("Read error while reading data file: " + dataFileName, e);
    } catch (SQLException e) {
        throw new DirectoryException(String.format("Table '%s' initialization failed: %s", table.getPhysicalName(), e.getMessage()), e);
    } finally {
        DirectoryException e = new DirectoryException();
        try {
            if (csvParser != null) {
                csvParser.close();
            }
        } catch (IOException ioe) {
            e.addSuppressed(ioe);
        }
        try {
            if (ps != null) {
                ps.close();
            }
        } catch (SQLException sqle) {
            e.addSuppressed(sqle);
        }
        if (e.getSuppressed().length > 0) {
            throw e;
        }
    }
}

public static final void buildMessagesProperties(String currentPath, String project, String filePath, List<String> activedLanguages, String defaultLanguage, String inputEncoding, String outputEncoding) {
    try {
        String newPath = currentPath + project + LoaderTranslation.PROPERTIES_PATH;
        File folderProject = new File(newPath);
        if (!folderProject.exists()) {
            folderProject.mkdirs();
        }
        InputStreamReader reader = new InputStreamReader(new FileInputStream(new File(filePath)), inputEncoding);
        LOG.info("File CSV encoding: " + inputEncoding);
        LOG.info("File properties encoding: " + outputEncoding);
        CSVFormat csvFormat = CSVFormat.DEFAULT;
        CSVParser readerCSV = new CSVParser(reader, csvFormat);
        List<CSVRecord> records = null;
        try {
            records = readerCSV.getRecords();
        } catch (Exception e) {
            LOG.error("Failed to load: " + filePath, e);
        }
        String prefixFileName = "";
        CSVRecord firstLineRecord = records.get(0);
        String[] prefixFileNameTemp = firstLineRecord.get(0).split("## Filename :");
        if (prefixFileNameTemp.length > 1) {
            prefixFileName = prefixFileNameTemp[1].trim();
        }
        String prefixKey = "";
        CSVRecord secondLineRecord = records.get(1);
        String[] prefixKeyTemp = secondLineRecord.get(0).split("## PrefixKey :");
        if (prefixKeyTemp.length > 1) {
            prefixKey = prefixKeyTemp[1].trim();
        }
        Map<String, Integer> availableLanguages = new HashMap<String, Integer>();
        for (CSVRecord record : records) {
            String firstCell = record.get(0);
            if (firstCell.contains("Prefix") && record.size() > 1) {
                String secondCell = record.get(1);
                if (secondCell.contains("Key")) {
                    for (int i = 2; i < record.size(); i++) {
                        String languageCode = record.get(i);
                        availableLanguages.put(languageCode, new Integer(i));
                    }
                    break;
                }
            }
        }
        // BUILD DEFAULT FILE
        String fileFullPath = newPath + "/" + prefixFileName + ".properties";
        Integer defaultLanguagePosition = availableLanguages.get(defaultLanguage);
        buildMessagesProperties(records, fileFullPath, prefixKey, defaultLanguage, defaultLanguagePosition, outputEncoding, true);
        for (Iterator<String> iterator = availableLanguages.keySet().iterator(); iterator.hasNext(); ) {
            String languageCode = (String) iterator.next();
            if (activedLanguages.contains(languageCode)) {
                Integer languagePosition = availableLanguages.get(languageCode);
                String languegFileFullPath = newPath + "/" + prefixFileName + "_" + languageCode + ".properties";
                buildMessagesProperties(records, languegFileFullPath, prefixKey, languageCode, languagePosition.intValue(), outputEncoding, false);
            }
        }
        LOG.info(newPath + "/" + prefixFileName + ".properties");
    } catch (Exception e) {
        LOG.info("Exception", e);
    }
}

public Map<String, List<String>> readTextFile(File textFile) throws Exception {
    Map<String, List<String>> ret = new HashMap<String, List<String>>();
    String delimiter = config.getUserSyncFileSourceDelimiter();
    CSVFormat csvFormat = CSVFormat.newFormat(delimiter.charAt(0));
    CSVParser csvParser = new CSVParser(new BufferedReader(new FileReader(textFile)), csvFormat);
    List<CSVRecord> csvRecordList = csvParser.getRecords();
    if (csvRecordList != null) {
        for (CSVRecord csvRecord : csvRecordList) {
            List<String> groups = new ArrayList<String>();
            String user = csvRecord.get(0);
            user = user.replaceAll("^\"|\"$", "");
            int i = csvRecord.size();
            for (int j = 1; j < i; j++) {
                String group = csvRecord.get(j);
                if (group != null && !group.isEmpty()) {
                    group = group.replaceAll("^\"|\"$", "");
                    groups.add(group);
                }
            }
            ret.put(user, groups);
        }
    }
    csvParser.close();
    return ret;
}

@Override
public Response apply(ContainerRequestContext ctx) {
    List<Parameter> parameters = operation.getParameters();
    final RequestContext requestContext = createContext(ctx);
    String path = ctx.getUriInfo().getPath();
    Map<String, Map<String, String>> formMap = new HashMap<String, Map<String, String>>();
    Map<String, File> inputStreams = new HashMap<String, File>();
    Object[] args = new Object[parameters.size() + 1];
    if (parameters != null) {
        int i = 0;
        args[i] = requestContext;
        i += 1;
        List<ValidationMessage> missingParams = new ArrayList<ValidationMessage>();
        UriInfo uri = ctx.getUriInfo();
        String formDataString = null;
        String[] parts = null;
        Set<String> existingKeys = new HashSet<String>();
        for (Iterator<String> x = uri.getQueryParameters().keySet().iterator(); x.hasNext(); ) {
            existingKeys.add(x.next() + ": qp");
        }
        for (Iterator<String> x = uri.getPathParameters().keySet().iterator(); x.hasNext(); ) {
            existingKeys.add(x.next() + ": pp");
        }
        for (Iterator<String> x = ctx.getHeaders().keySet().iterator(); x.hasNext(); ) {
            String key = x.next();
        //              if(!commonHeaders.contains(key))
        //                existingKeys.add(key);
        }
        MediaType mt = requestContext.getMediaType();
        for (Parameter p : parameters) {
            Map<String, String> headers = new HashMap<String, String>();
            String name = null;
            if (p instanceof FormParameter) {
                if (formDataString == null) {
                    // can only read stream once
                    if (mt.isCompatible(MediaType.MULTIPART_FORM_DATA_TYPE)) {
                        // get the boundary
                        String boundary = mt.getParameters().get("boundary");
                        if (boundary != null) {
                            try {
                                InputStream output = ctx.getEntityStream();
                                MultipartStream multipartStream = new MultipartStream(output, boundary.getBytes());
                                boolean nextPart = multipartStream.skipPreamble();
                                while (nextPart) {
                                    String header = multipartStream.readHeaders();
                                    // process headers
                                    if (header != null) {
                                        CSVFormat format = CSVFormat.DEFAULT.withDelimiter(';').withRecordSeparator("=");
                                        Iterable<CSVRecord> records = format.parse(new StringReader(header));
                                        for (CSVRecord r : records) {
                                            for (int j = 0; j < r.size(); j++) {
                                                String string = r.get(j);
                                                Iterable<CSVRecord> outerString = CSVFormat.DEFAULT.withDelimiter('=').parse(new StringReader(string));
                                                for (CSVRecord outerKvPair : outerString) {
                                                    if (outerKvPair.size() == 2) {
                                                        String key = outerKvPair.get(0).trim();
                                                        String value = outerKvPair.get(1).trim();
                                                        if ("name".equals(key)) {
                                                            name = value;
                                                        }
                                                        headers.put(key, value);
                                                    } else {
                                                        Iterable<CSVRecord> innerString = CSVFormat.DEFAULT.withDelimiter(':').parse(new StringReader(string));
                                                        for (CSVRecord innerKVPair : innerString) {
                                                            if (innerKVPair.size() == 2) {
                                                                String key = innerKVPair.get(0).trim();
                                                                String value = innerKVPair.get(1).trim();
                                                                if ("name".equals(key)) {
                                                                    name = value;
                                                                }
                                                                headers.put(key, value);
                                                            }
                                                        }
                                                    }
                                                }
                                                if (name != null) {
                                                    formMap.put(name, headers);
                                                }
                                            }
                                        }
                                    }
                                    String filename = extractFilenameFromHeaders(headers);
                                    if (filename != null) {
                                        try {
                                            File file = new File(Files.createTempDir(), filename);
                                            file.deleteOnExit();
                                            file.getParentFile().deleteOnExit();
                                            FileOutputStream fo = new FileOutputStream(file);
                                            multipartStream.readBodyData(fo);
                                            inputStreams.put(name, file);
                                        } catch (Exception e) {
                                            LOGGER.error("Failed to extract uploaded file", e);
                                        }
                                    } else {
                                        ByteArrayOutputStream bo = new ByteArrayOutputStream();
                                        multipartStream.readBodyData(bo);
                                        String value = bo.toString();
                                        headers.put(name, value);
                                    }
                                    if (name != null) {
                                        formMap.put(name, headers);
                                    }
                                    headers = new HashMap<>();
                                    name = null;
                                    nextPart = multipartStream.readBoundary();
                                }
                            } catch (IOException e) {
                                e.printStackTrace();
                            }
                        }
                    } else {
                        try {
                            formDataString = IOUtils.toString(ctx.getEntityStream(), "UTF-8");
                            parts = formDataString.split("&");
                            for (String part : parts) {
                                String[] kv = part.split("=");
                                existingKeys.add(kv[0] + ": fp");
                            }
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        }
        for (Parameter parameter : parameters) {
            String in = parameter.getIn();
            Object o = null;
            try {
                if ("formData".equals(in)) {
                    SerializableParameter sp = (SerializableParameter) parameter;
                    String name = parameter.getName();
                    if (mt.isCompatible(MediaType.MULTIPART_FORM_DATA_TYPE)) {
                        // look in the form map
                        Map<String, String> headers = formMap.get(name);
                        if (headers != null && headers.size() > 0) {
                            if ("file".equals(sp.getType())) {
                                o = inputStreams.get(name);
                            } else {
                                Object obj = headers.get(parameter.getName());
                                if (obj != null) {
                                    JavaType jt = parameterClasses[i];
                                    Class<?> cls = jt.getRawClass();
                                    List<String> os = Arrays.asList(obj.toString());
                                    try {
                                        o = validator.convertAndValidate(os, parameter, cls, definitions);
                                    } catch (ConversionException e) {
                                        missingParams.add(e.getError());
                                    } catch (ValidationException e) {
                                        missingParams.add(e.getValidationMessage());
                                    }
                                }
                            }
                        }
                    } else {
                        if (formDataString != null) {
                            for (String part : parts) {
                                String[] kv = part.split("=");
                                if (kv != null) {
                                    if (kv.length > 0) {
                                        existingKeys.remove(kv[0] + ": fp");
                                    }
                                    if (kv.length == 2) {
                                        // TODO how to handle arrays here?
                                        String key = kv[0];
                                        try {
                                            String value = URLDecoder.decode(kv[1], "utf-8");
                                            if (parameter.getName().equals(key)) {
                                                JavaType jt = parameterClasses[i];
                                                Class<?> cls = jt.getRawClass();
                                                try {
                                                    o = validator.convertAndValidate(Arrays.asList(value), parameter, cls, definitions);
                                                } catch (ConversionException e) {
                                                    missingParams.add(e.getError());
                                                } catch (ValidationException e) {
                                                    missingParams.add(e.getValidationMessage());
                                                }
                                            }
                                        } catch (UnsupportedEncodingException e) {
                                            LOGGER.error("unable to decode value for " + key);
                                        }
                                    }
                                }
                            }
                        }
                    }
                } else {
                    try {
                        String paramName = parameter.getName();
                        if ("query".equals(in)) {
                            existingKeys.remove(paramName + ": qp");
                        }
                        if ("path".equals(in)) {
                            existingKeys.remove(paramName + ": pp");
                        }
                        JavaType jt = parameterClasses[i];
                        Class<?> cls = jt.getRawClass();
                        if ("body".equals(in)) {
                            if (ctx.hasEntity()) {
                                BodyParameter body = (BodyParameter) parameter;
                                o = EntityProcessorFactory.readValue(ctx.getMediaType(), ctx.getEntityStream(), cls);
                                if (o != null) {
                                    validate(o, body.getSchema(), SchemaValidator.Direction.INPUT);
                                }
                            } else if (parameter.getRequired()) {
                                ValidationException e = new ValidationException();
                                e.message(new ValidationMessage().message("The input body `" + paramName + "` is required"));
                                throw e;
                            }
                        }
                        if ("query".equals(in)) {
                            o = validator.convertAndValidate(uri.getQueryParameters().get(parameter.getName()), parameter, cls, definitions);
                        } else if ("path".equals(in)) {
                            o = validator.convertAndValidate(uri.getPathParameters().get(parameter.getName()), parameter, cls, definitions);
                        } else if ("header".equals(in)) {
                            o = validator.convertAndValidate(ctx.getHeaders().get(parameter.getName()), parameter, cls, definitions);
                        }
                    } catch (ConversionException e) {
                        missingParams.add(e.getError());
                    } catch (ValidationException e) {
                        missingParams.add(e.getValidationMessage());
                    }
                }
            } catch (NumberFormatException e) {
                LOGGER.error("Couldn't find " + parameter.getName() + " (" + in + ") to " + parameterClasses[i], e);
            }
            args[i] = o;
            i += 1;
        }
        if (existingKeys.size() > 0) {
            LOGGER.debug("unexpected keys: " + existingKeys);
        }
        if (missingParams.size() > 0) {
            StringBuilder builder = new StringBuilder();
            builder.append("Input error");
            if (missingParams.size() > 1) {
                builder.append("s");
            }
            builder.append(": ");
            int count = 0;
            for (ValidationMessage message : missingParams) {
                if (count > 0) {
                    builder.append(", ");
                }
                if (message != null && message.getMessage() != null) {
                    builder.append(message.getMessage());
                } else {
                    builder.append("no additional input");
                }
                count += 1;
            }
            int statusCode = config.getInvalidRequestStatusCode();
            ApiError error = new ApiError().code(statusCode).message(builder.toString());
            throw new ApiException(error);
        }
    }
    try {
        if (method != null) {
            LOGGER.info("calling method " + method + " on controller " + this.controller + " with args " + Arrays.toString(args));
            try {
                Object response = method.invoke(controller, args);
                if (response instanceof ResponseContext) {
                    ResponseContext wrapper = (ResponseContext) response;
                    ResponseBuilder builder = Response.status(wrapper.getStatus());
                    // response headers
                    for (String key : wrapper.getHeaders().keySet()) {
                        List<String> v = wrapper.getHeaders().get(key);
                        if (v.size() == 1) {
                            builder.header(key, v.get(0));
                        } else {
                            builder.header(key, v);
                        }
                    }
                    // entity
                    if (wrapper.getEntity() != null) {
                        builder.entity(wrapper.getEntity());
                        // content type
                        if (wrapper.getContentType() != null) {
                            builder.type(wrapper.getContentType());
                        } else {
                            final ContextResolver<ContentTypeSelector> selector = providersProvider.get().getContextResolver(ContentTypeSelector.class, MediaType.WILDCARD_TYPE);
                            if (selector != null) {
                                selector.getContext(getClass()).apply(ctx.getAcceptableMediaTypes(), builder);
                            }
                        }
                        if (operation.getResponses() != null) {
                            String responseCode = String.valueOf(wrapper.getStatus());
                            io.swagger.models.Response responseSchema = operation.getResponses().get(responseCode);
                            if (responseSchema == null) {
                                // try default response schema
                                responseSchema = operation.getResponses().get("default");
                            }
                            if (responseSchema != null && responseSchema.getSchema() != null) {
                                validate(wrapper.getEntity(), responseSchema.getSchema(), SchemaValidator.Direction.OUTPUT);
                            } else {
                                LOGGER.debug("no response schema for code " + responseCode + " to validate against");
                            }
                        }
                    }
                    return builder.build();
                }
                return Response.ok().entity(response).build();
            } catch (IllegalArgumentExceptionIllegalAccessException | InvocationTargetException |  e) {
                for (Throwable cause = e.getCause(); cause != null; ) {
                    if (cause instanceof ApiException) {
                        throw (ApiException) cause;
                    }
                    final Throwable next = cause.getCause();
                    cause = next == cause || next == null ? null : next;
                }
                throw new ApiException(ApiErrorUtils.createInternalError(), e);
            }
        }
        Map<String, io.swagger.models.Response> responses = operation.getResponses();
        if (responses != null) {
            String[] keys = new String[responses.keySet().size()];
            Arrays.sort(responses.keySet().toArray(keys));
            int code = 0;
            String defaultKey = null;
            for (String key : keys) {
                if (key.startsWith("2")) {
                    defaultKey = key;
                    code = Integer.parseInt(key);
                    break;
                }
                if ("default".equals(key)) {
                    defaultKey = key;
                    code = 200;
                    break;
                }
                if (key.startsWith("3")) {
                    // we use the 3xx responses as defaults
                    defaultKey = key;
                    code = Integer.parseInt(key);
                }
            }
            if (defaultKey != null) {
                ResponseBuilder builder = Response.status(code);
                io.swagger.models.Response response = responses.get(defaultKey);
                if (response.getHeaders() != null && response.getHeaders().size() > 0) {
                    for (String key : response.getHeaders().keySet()) {
                        Property headerProperty = response.getHeaders().get(key);
                        Object output = ExampleBuilder.fromProperty(headerProperty, definitions);
                        if (output instanceof ArrayExample) {
                            output = ((ArrayExample) output).asString();
                        } else if (output instanceof ObjectExample) {
                            LOGGER.debug("not serializing output example, only primitives or arrays of primitives are supported");
                        } else {
                            output = ((Example) output).asString();
                        }
                        builder.header(key, output);
                    }
                }
                Map<String, Object> examples = response.getExamples();
                if (examples != null) {
                    for (MediaType mediaType : requestContext.getAcceptableMediaTypes()) {
                        for (String key : examples.keySet()) {
                            if (MediaType.valueOf(key).isCompatible(mediaType)) {
                                builder.entity(examples.get(key)).type(mediaType);
                                return builder.build();
                            }
                        }
                    }
                }
                Object output = ExampleBuilder.fromProperty(response.getSchema(), definitions);
                if (output != null) {
                    ResponseContext resp = new ResponseContext().entity(output);
                    setContentType(requestContext, resp, operation);
                    builder.entity(output);
                    if (resp.getContentType() != null) {
                        // this comes from the operation itself
                        builder.type(resp.getContentType());
                    } else {
                        // get acceptable content types
                        List<EntityProcessor> processors = EntityProcessorFactory.getProcessors();
                        MediaType responseMediaType = null;
                        // take first compatible one
                        for (EntityProcessor processor : processors) {
                            if (responseMediaType != null) {
                                break;
                            }
                            for (MediaType mt : requestContext.getAcceptableMediaTypes()) {
                                LOGGER.debug("checking type " + mt.toString() + " against " + processor.getClass().getName());
                                if (processor.supports(mt)) {
                                    builder.type(mt);
                                    responseMediaType = mt;
                                    break;
                                }
                            }
                        }
                        if (responseMediaType == null) {
                            // no match based on Accept header, use first processor in list
                            for (EntityProcessor processor : processors) {
                                List<MediaType> supportedTypes = processor.getSupportedMediaTypes();
                                if (supportedTypes.size() > 0) {
                                    builder.type(supportedTypes.get(0));
                                    break;
                                }
                            }
                        }
                    }
                    builder.entity(output);
                }
                return builder.build();
            } else {
                LOGGER.debug("no response type to map to, assume 200");
                code = 200;
            }
            return Response.status(code).build();
        }
        return Response.ok().build();
    } finally {
        for (String key : inputStreams.keySet()) {
            File file = inputStreams.get(key);
            if (file != null) {
                LOGGER.debug("deleting file " + file.getPath());
                file.delete();
            }
        }
    }
}

public static void parseStudy(InputStream stream, XHTMLContentHandler xhtml, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException {
    TikaInputStream tis = TikaInputStream.get(stream);
    // Automatically detect the character encoding
    TikaConfig tikaConfig = context.get(TikaConfig.class);
    if (tikaConfig == null) {
        tikaConfig = TikaConfig.getDefaultConfig();
    }
    try (AutoDetectReader reader = new AutoDetectReader(new CloseShieldInputStream(tis), metadata, tikaConfig.getEncodingDetector());
        CSVParser csvParser = new CSVParser(reader, CSVFormat.TDF)) {
        Iterator<CSVRecord> iterator = csvParser.iterator();
        xhtml.startElement("table");
        xhtml.startElement("thead");
        if (iterator.hasNext()) {
            CSVRecord record = iterator.next();
            for (int i = 0; i < record.size(); i++) {
                xhtml.startElement("th");
                xhtml.characters(record.get(i));
                xhtml.endElement("th");
            }
        }
        xhtml.endElement("thead");
        xhtml.startElement("tbody");
        while (iterator.hasNext()) {
            CSVRecord record = iterator.next();
            xhtml.startElement("tr");
            for (int j = 0; j < record.size(); j++) {
                xhtml.startElement("td");
                xhtml.characters(record.get(j));
                xhtml.endElement("td");
            }
            xhtml.endElement("tr");
        }
        xhtml.endElement("tbody");
        xhtml.endElement("table");
    }
}

public Importer onSuccessFromCsvForm() {
    CSVParser parser = null;
    parsed = new Vector<Stock>();
    try {
        parser = new CSVParser(new StringReader(rawcsv), CSVFormat.EXCEL.withHeader());
        for (CSVRecord rec : parser) {
            Stock stock = recordToStock(rec);
            if (stock != null) {
                parsed.add(stock);
            }
        }
    } catch (Exception e) {
    }
    try {
        parser.close();
    } catch (Exception e) {
    }
    return this;
}

/**
     * Returns an IData representation of the CSV data in the given input stream.
     *
     * @param inputStream The input stream to be decoded.
     * @param charset     The character set to use.
     * @return An IData representation of the given input stream data.
     * @throws IOException If there is a problem reading from the stream.
     */
@Override
public IData decode(InputStream inputStream, Charset charset) throws IOException {
    if (inputStream == null)
        return null;
    Reader reader = new InputStreamReader(inputStream, CharsetHelper.normalize(charset));
    CSVFormat format = CSVFormat.DEFAULT.withHeader().withDelimiter(delimiter).withNullString("");
    CSVParser parser = format.parse(reader);
    Set<String> keys = parser.getHeaderMap().keySet();
    List<IData> list = new ArrayList<IData>();
    for (CSVRecord record : parser) {
        IData document = IDataFactory.create();
        IDataCursor cursor = document.getCursor();
        for (String key : keys) {
            if (record.isSet(key)) {
                String value = record.get(key);
                if (value != null)
                    IDataUtil.put(cursor, key, value);
            }
        }
        cursor.destroy();
        list.add(document);
    }
    IData output = IDataFactory.create();
    IDataCursor cursor = output.getCursor();
    IDataUtil.put(cursor, "recordWithNoID", list.toArray(new IData[list.size()]));
    return output;
}

public static Map<String, Pattern> importPatterns(String inputDir, Integer length) {
    Map<String, Pattern> patterns = new HashMap<String, Pattern>();
    //		CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator("\n").
    //				withIgnoreEmptyLines(true).
    //				withDelimiter(' ').
    //				withIgnoreSurroundingSpaces(true).
    //				withHeader();
    CSVFormat csvFileFormat = CSVFormat.RFC4180.withHeader();
    CSVParser csvParserTotalFrequency;
    CSVParser csvParser;
    File patternDir = new File(inputDir);
    if (patternDir.exists()) {
        File[] patternFiles = patternDir.listFiles();
        for (File file : patternFiles) {
            try {
                if (file.isDirectory())
                    continue;
                csvParser = CSVParser.parse(file, Charsets.UTF_8, csvFileFormat);
                Map<String, Integer> headerMap = csvParser.getHeaderMap();
                csvParserTotalFrequency = CSVParser.parse(file, Charsets.UTF_8, csvFileFormat);
                int totalFrequency = getTotalFrequency(csvParserTotalFrequency, headerMap.get(COUNT_COLUMN));
                for (CSVRecord record : csvParser) {
                    Pattern p = getPattern(record, headerMap, totalFrequency);
                    if (p != null) {
                        if (length != null) {
                            if (p.getLength() == length.intValue())
                                patterns.put(p.getId(), p);
                        } else {
                            patterns.put(p.getId(), p);
                        }
                    //							System.out.println(p.getPrintStr());
                    } else {
                        logger.error("Error in reading the pattern: " + record.toString());
                    }
                }
            } catch (IOException e) {
                logger.error("Error in reading the patterns from the file " + file.getAbsolutePath());
                e.printStackTrace();
            }
        }
    }
    return patterns;
}

private boolean createLocationsCsv() throws Throwable {
    try {
        open();
        final String fileName = quotedVal();
        if (fileName == null) {
            error("Expected csv filename");
            return false;
        }
        final File csvf = new File(fileName);
        if (!csvf.exists()) {
            error("File " + fileName + " does not exist");
            return false;
        }
        final CSVParser parser = CSVFormat.DEFAULT.withHeader().withAllowMissingColumnNames().parse(new FileReader(csvf));
        for (final CSVRecord rec : parser) {
            final BwLocation loc = BwLocation.makeLocation();
            String fld = rec.get("Address");
            if (fld == null) {
                error("address required for location");
                continue;
            }
            loc.setAddressField(fld);
            fld = rec.get("Room");
            if (fld != null) {
                loc.setRoomField(fld);
            }
            fld = rec.get("Code");
            if (fld != null) {
                loc.setSubField1(fld);
            }
            /*        fld = rec.get("subField2");
        if (fld != null) {
          loc.setSubField1(fld);
        }*/
            fld = rec.get("Accessible");
            if (fld != null) {
                loc.setAccessible("Y".equals(fld));
            }
            fld = rec.get("Geouri");
            if (fld != null) {
                loc.setGeouri(fld);
            }
            fld = rec.get("Street");
            if (fld != null) {
                loc.setStreet(fld);
            }
            fld = rec.get("City");
            if (fld != null) {
                loc.setCity(fld);
            }
            fld = rec.get("State");
            if (fld != null) {
                loc.setState(fld);
            }
            fld = rec.get("Zip");
            if (fld != null) {
                loc.setZip(fld);
            }
            fld = rec.get("Link");
            if (fld != null) {
                loc.setLink(fld);
            }
            getSvci().getLocationsHandler().add(loc);
        }
        return true;
    } finally {
        close();
    }
}

@Override
public void configure(OETLProcessor iProcessor, ODocument iConfiguration, OCommandContext iContext) {
    super.configure(iProcessor, iConfiguration, iContext);
    csvFormat = CSVFormat.newFormat(',').withNullString(NULL_STRING).withEscape('\\').withQuote('"').withCommentMarker('#');
    if (iConfiguration.containsField("predefinedFormat")) {
        csvFormat = CSVFormat.valueOf(iConfiguration.<String>field("predefinedFormat"));
    }
    if (iConfiguration.containsField("separator")) {
        csvFormat = csvFormat.withDelimiter(iConfiguration.field("separator").toString().charAt(0));
    }
    if (iConfiguration.containsField("dateFormat")) {
        dateFormat = iConfiguration.<String>field("dateFormat");
    }
    if (iConfiguration.containsField("dateTimeFormat")) {
        dateFormat = iConfiguration.<String>field("dateTimeFormat");
    }
    if (iConfiguration.containsField("ignoreEmptyLines")) {
        boolean ignoreEmptyLines = iConfiguration.field("ignoreEmptyLines");
        csvFormat = csvFormat.withIgnoreEmptyLines(ignoreEmptyLines);
    }
    if (iConfiguration.containsField("ignoreMissingColumns")) {
        boolean ignoreMissingColumns = iConfiguration.field("ignoreMissingColumns");
        csvFormat = csvFormat.withAllowMissingColumnNames(ignoreMissingColumns);
    }
    if (iConfiguration.containsField("columnsOnFirstLine")) {
        Boolean columnsOnFirstLine = (Boolean) iConfiguration.field("columnsOnFirstLine");
        if (columnsOnFirstLine.equals(Boolean.TRUE)) {
            csvFormat = csvFormat.withHeader();
        }
    } else {
        csvFormat = csvFormat.withHeader();
    }
    if (iConfiguration.containsField("columns")) {
        final List<String> columns = iConfiguration.field("columns");
        ArrayList<String> columnNames = new ArrayList<String>(columns.size());
        columnTypes = new HashMap<String, OType>(columns.size());
        for (String c : columns) {
            final String[] parts = c.split(":");
            columnNames.add(parts[0]);
            if (parts.length > 1) {
                columnTypes.put(parts[0], OType.valueOf(parts[1].toUpperCase()));
            } else {
                columnTypes.put(parts[0], OType.ANY);
            }
        }
        log(OETLProcessor.LOG_LEVELS.INFO, "column types: %s", columnTypes);
        csvFormat = csvFormat.withHeader(columnNames.toArray(new String[] {})).withSkipHeaderRecord(true);
    }
    if (iConfiguration.containsField("skipFrom")) {
        skipFrom = ((Number) iConfiguration.field("skipFrom")).longValue();
    }
    if (iConfiguration.containsField("skipTo")) {
        skipTo = ((Number) iConfiguration.field("skipTo")).longValue();
    }
    if (iConfiguration.containsField("nullValue")) {
        nullValue = iConfiguration.<String>field("nullValue");
        csvFormat = csvFormat.withNullString(nullValue);
    }
    if (iConfiguration.containsField("quote")) {
        final String value = iConfiguration.field("quote").toString();
        if (!value.isEmpty()) {
            csvFormat = csvFormat.withQuote(value.charAt(0));
        }
    }
}

private List<UserCSV> getUsersFromRequest(PortletRequest request, String fileUploadFormId) throws IOException, TException {
    final UploadPortletRequest uploadPortletRequest = PortalUtil.getUploadPortletRequest(request);
    final InputStream stream = uploadPortletRequest.getFileAsStream(fileUploadFormId);
    Reader reader = new InputStreamReader(stream);
    CSVFormat format = CommonUtils.sw360CsvFormat;
    CSVParser parser = new CSVParser(reader, format);
    List<CSVRecord> records;
    records = parser.getRecords();
    if (records.size() > 0) {
        // Remove header
        records.remove(0);
    }
    return getUsersFromCSV(records);
}

@Override
public ArrayNode checkFile(InputStream rulesFileStream) throws IOException, ReferentialException, InvalidParseOperationException {
    ParametersChecker.checkParameter(RULES_FILE_STREAM_IS_A_MANDATORY_PARAMETER, rulesFileStream);
    if (!isCollectionEmptyForTenant()) {
        throw new FileRulesException(String.format(RULES_COLLECTION_IS_NOT_EMPTY, getTenant()));
    }
    File csvFileReader = convertInputStreamToFile(rulesFileStream);
    try (FileReader reader = new FileReader(csvFileReader)) {
        @SuppressWarnings("resource") final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.withHeader());
        final HashSet<String> ruleIdSet = new HashSet<>();
        for (final CSVRecord record : parser) {
            try {
                if (checkRecords(record)) {
                    final String ruleId = record.get(RULEID);
                    final String ruleType = record.get(RULE_TYPE);
                    final String ruleValue = record.get(RULE_VALUE);
                    final String ruleDuration = record.get(RULE_DURATION);
                    final String ruleMeasurementValue = record.get(RULE_MEASUREMENT);
                    checkParametersNotEmpty(ruleId, ruleType, ruleValue, ruleDuration, ruleMeasurementValue);
                    checkRuleDurationIsInteger(ruleDuration);
                    if (ruleIdSet.contains(ruleId)) {
                        throw new FileRulesException(String.format(FILE_RULE_WITH_RULE_ID, ruleId));
                    }
                    ruleIdSet.add(ruleId);
                    if (!contains(ruleMeasurementValue)) {
                        throw new FileRulesException(String.format(NOT_SUPPORTED_VALUE, RULE_MEASUREMENT, ruleMeasurementValue));
                    }
                }
            } catch (final Exception e) {
                throw new FileRulesException(INVALID_CSV_FILE + e.getMessage());
            }
        }
    }
    if (csvFileReader != null) {
        final ArrayNode readRulesAsJson = RulesManagerParser.readObjectsFromCsvWriteAsArrayNode(csvFileReader);
        csvFileReader.delete();
        return readRulesAsJson;
    }
    /* this line is reached only if temporary file is null */
    throw new FileRulesException(INVALID_CSV_FILE);
}

@Override
public void process(OutputStream stream, AisPacket packet, long count) throws IOException {
    csvLock.lock();
    try {
        if (csv == null) {
            csv = new CSVPrinter(new OutputStreamWriter(stream), CSVFormat.DEFAULT.withHeader(columns));
        }
        AisMessage m = packet.tryGetAisMessage();
        if (m != null) {
            List fields = Lists.newArrayList();
            for (String column : columns) {
                switch(column) {
                    case "timestamp":
                        addTimestamp(fields, packet);
                        break;
                    case "timestamp_pretty":
                        addTimestampPretty(fields, packet);
                        break;
                    case "msgid":
                        addMsgId(fields, m);
                        break;
                    case "mmsi":
                        addMmsi(fields, m);
                        break;
                    case "posacc":
                        addPositionAccuracy(fields, m);
                        break;
                    case "lat":
                        addLatitude(fields, m);
                        break;
                    case "lon":
                        addLongitude(fields, m);
                        break;
                    case "targetType":
                        addTargetType(fields, m);
                        break;
                    case "sog":
                        addSog(fields, m);
                        break;
                    case "cog":
                        addCog(fields, m);
                        break;
                    case "trueHeading":
                        addTrueHeading(fields, m);
                        break;
                    case "draught":
                        addDraught(fields, m);
                        break;
                    case "name":
                        addName(fields, m);
                        break;
                    case "dimBow":
                        addDimBow(fields, m);
                        break;
                    case "dimPort":
                        addDimPort(fields, m);
                        break;
                    case "dimStarboard":
                        addDimStarboard(fields, m);
                        break;
                    case "dimStern":
                        addDimStern(fields, m);
                        break;
                    case "shipTypeCargoTypeCode":
                        addShipTypeCargoTypeCode(fields, m);
                        break;
                    case "shipType":
                        addShipType(fields, m);
                        break;
                    case "shipCargo":
                        addShipCargo(fields, m);
                        break;
                    case "callsign":
                        addCallsign(fields, m);
                        break;
                    case "imo":
                        addImo(fields, m);
                        break;
                    case "destination":
                        addDestination(fields, m);
                        break;
                    case "eta":
                        addEta(fields, m);
                        break;
                    default:
                        LOG.warn("Unknown column: " + column);
                        fields.add(NULL_INDICATOR);
                }
            }
            csv.printRecord(fields);
        }
    } finally {
        csvLock.unlock();
    }
}

@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    // Set the default merge mode
    boolean merge = true;
    // Defines the interval in minutes that will be used to merge encounters.
    int interval = DEFAULT_INTERVAL_MINS;
    String intervalParameter = request.getParameter("interval");
    if (intervalParameter != null) {
        int newInterval = Integer.valueOf(intervalParameter);
        if (newInterval >= 0) {
            interval = newInterval;
            if (interval == 0) {
                merge = false;
            }
        } else {
            log.error("Interval value is less then 0. Default used.");
        }
    }
    CSVPrinter printer = new CSVPrinter(response.getWriter(), CSVFormat.EXCEL.withDelimiter(','));
    //check for authenticated users
    if (!XformsUtil.isAuthenticated(request, response, null))
        return;
    Date now = new Date();
    DateFormat format = new SimpleDateFormat("yyyyMMdd_HHmmss");
    String filename = String.format("buendiadata_%s.csv", format.format(now));
    String contentDispositionHeader = String.format("attachment; filename=%s;", filename);
    response.addHeader("Content-Disposition", contentDispositionHeader);
    PatientService patientService = Context.getPatientService();
    EncounterService encounterService = Context.getEncounterService();
    List<Patient> patients = new ArrayList<>(patientService.getAllPatients());
    Collections.sort(patients, PATIENT_COMPARATOR);
    // We may want to get the observations displayed in the chart/xform, in which case there
    // are a few
    // sensible orders:
    // 1: UUID
    // 2: Order in chart
    // 3: Order in Xform
    // Order in Xform/chart is not good as stuff changes every time we change xform
    // So instead we will use UUID order, but use the Chart form to use the concepts to display.
    Set<Concept> questionConcepts = new HashSet<>();
    for (Form form : ChartResource.getCharts(Context.getFormService())) {
        TreeMap<Integer, TreeSet<FormField>> formStructure = FormUtil.getFormStructure(form);
        for (FormField groupField : formStructure.get(0)) {
            for (FormField fieldInGroup : formStructure.get(groupField.getId())) {
                questionConcepts.add(fieldInGroup.getField().getConcept());
            }
        }
    }
    FixedSortedConceptIndexer indexer = new FixedSortedConceptIndexer(questionConcepts);
    // Write English headers.
    writeHeaders(printer, indexer);
    Calendar calendar = Calendar.getInstance();
    // Loop through all the patients and get their encounters.
    for (Patient patient : patients) {
        // Define an array that will represent the line that will be inserted in the CSV.
        Object[] previousCSVLine = new Object[FIXED_HEADERS.length + indexer.size() * COLUMNS_PER_OBS];
        Date deadLine = new Date(0);
        ArrayList<Encounter> encounters = new ArrayList<>(encounterService.getEncountersByPatient(patient));
        Collections.sort(encounters, ENCOUNTER_COMPARATOR);
        // TODO: For now patients with no encounters are ignored. List them on the future.
        if (encounters.size() == 0)
            continue;
        // Loop through all the encounters for this patient to get the observations.
        for (Encounter encounter : encounters) {
            try {
                // Flag to whether we will use the merged version of the encounter
                // or the single version.
                boolean useMerged = merge;
                // Array that will be used to merge in previous encounter with the current one.
                Object[] mergedCSVLine = new Object[previousCSVLine.length];
                // Duplicate previous encounter into the (future to be) merged one.
                System.arraycopy(previousCSVLine, 0, mergedCSVLine, 0, previousCSVLine.length);
                // Define the array to be used to store the current encounter.
                Object[] currentCSVLine = new Object[FIXED_HEADERS.length + indexer.size() * COLUMNS_PER_OBS];
                // If the current encounter is more then "interval" minutes from the previous
                // print the previous and reset it.
                Date encounterTime = encounter.getEncounterDatetime();
                if (encounterTime.after(deadLine)) {
                    printer.printRecord(previousCSVLine);
                    previousCSVLine = new Object[FIXED_HEADERS.length + indexer.size() * COLUMNS_PER_OBS];
                    useMerged = false;
                }
                // Set the next deadline as the current encounter time plus "interval" minutes.
                calendar.setTime(encounterTime);
                calendar.add(Calendar.MINUTE, interval);
                deadLine = calendar.getTime();
                // Fill the fixed columns values.
                currentCSVLine[0] = patient.getUuid();
                currentCSVLine[1] = patient.getPatientIdentifier("MSF");
                if (patient.getBirthdate() != null) {
                    currentCSVLine[2] = Utils.YYYYMMDD_UTC_FORMAT.format(patient.getBirthdate());
                }
                currentCSVLine[3] = encounter.getUuid();
                currentCSVLine[4] = encounterTime.getTime();
                currentCSVLine[5] = Utils.toIso8601(encounterTime);
                currentCSVLine[6] = Utils.SPREADSHEET_FORMAT.format(encounterTime);
                // All the values fo the fixed columns saved in the current encounter line
                // will also be saved to the merged line.
                System.arraycopy(currentCSVLine, 0, mergedCSVLine, 0, 7);
                // Loop through all the observations for this encounter
                for (Obs obs : encounter.getAllObs()) {
                    Integer index = indexer.getIndex(obs.getConcept());
                    if (index == null)
                        continue;
                    // For each observation there are three columns: if the value of the
                    // observation is a concept, then the three columns contain the English
                    // name, the OpenMRS ID, and the UUID of the concept; otherwise all
                    // three columns contain the formatted value.
                    int valueColumn = FIXED_HEADERS.length + index * COLUMNS_PER_OBS;
                    // Coded values are treated differently
                    if (obs.getValueCoded() != null) {
                        Concept value = obs.getValueCoded();
                        currentCSVLine[valueColumn] = NAMER.getClientName(value);
                        currentCSVLine[valueColumn + 1] = value.getId();
                        currentCSVLine[valueColumn + 2] = value.getUuid();
                        if (useMerged) {
                            // If we are still merging the current encounter values into
                            // the previous one get the previous value and see if it had
                            // something in it.
                            String previousValue = (String) mergedCSVLine[valueColumn];
                            if ((previousValue == null) || (previousValue.isEmpty())) {
                                // If the previous value was empty copy the current value into it.
                                mergedCSVLine[valueColumn] = currentCSVLine[valueColumn];
                                mergedCSVLine[valueColumn + 1] = currentCSVLine[valueColumn + 1];
                                mergedCSVLine[valueColumn + 2] = currentCSVLine[valueColumn + 2];
                            } else {
                                // If the previous encounter have values stored for this
                                // observation we cannot merge them anymore.
                                useMerged = false;
                            }
                        }
                    } else // All values except the coded ones will be treated equally.
                    {
                        // Return the value of the the current observation using the visitor.
                        String value = (String) VisitObsValue.visit(obs, stringVisitor);
                        // Check if we have values stored for this observation
                        if ((value != null) && (!value.isEmpty())) {
                            // Save the value of the observation on the current encounter line.
                            currentCSVLine[valueColumn] = value;
                            currentCSVLine[valueColumn + 1] = value;
                            currentCSVLine[valueColumn + 2] = value;
                            if (useMerged) {
                                // Since we are still merging this encounter with the previous
                                // one let's get the previous value to see if it had something
                                // stored on it.
                                String previousValue = (String) mergedCSVLine[valueColumn];
                                if ((previousValue != null) && (!previousValue.isEmpty())) {
                                    // the previous encounter
                                    if (obs.getValueText() != null) {
                                        // We only continue merging if the observation is of
                                        // type text, so we concatenate it.
                                        // TODO: add timestamps to the merged values that are of type text
                                        previousValue += "\n" + value;
                                        value = previousValue;
                                    } else {
                                        // Any other type of value we stop the merging.
                                        useMerged = false;
                                    }
                                }
                                mergedCSVLine[valueColumn] = value;
                                mergedCSVLine[valueColumn + 1] = value;
                                mergedCSVLine[valueColumn + 2] = value;
                            }
                        }
                    }
                }
                if (useMerged) {
                    // If after looping through all the observations we didn't had any
                    // overlapped values we keep the merged line.
                    previousCSVLine = mergedCSVLine;
                } else {
                    // current encounter the previous one. Only if the previous line is not empty.
                    if (previousCSVLine[0] != null) {
                        printer.printRecord(previousCSVLine);
                    }
                    previousCSVLine = currentCSVLine;
                }
            } catch (Exception e) {
                log.error("Error exporting encounter", e);
            }
        }
        // For the last encounter we print the remaining line.
        printer.printRecord(previousCSVLine);
    }
}

private void setupCsvToFeatureIterator(final File csvFile, final long startLine, final WRS2GeometryStore geometryStore, final Filter cqlFilter) throws FileNotFoundException, IOException {
    parserFis = new FileInputStream(csvFile);
    parserIsr = new InputStreamReader(parserFis, StringUtils.UTF8_CHAR_SET);
    parser = new CSVParser(parserIsr, CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord());
    final Iterator<CSVRecord> csvIterator = parser.iterator();
    long startLineDecrementor = startLine;
    // we skip the header, so only skip to start line 1
    while ((startLineDecrementor > 1) && csvIterator.hasNext()) {
        startLineDecrementor--;
        csvIterator.next();
    }
    // wrap the iterator with a feature conversion and a filter (if
    // provided)
    iterator = Iterators.transform(csvIterator, new CSVToFeatureTransform(geometryStore, type));
    if (cqlFilter != null) {
        Filter actualFilter;
        if (hasOtherProperties(cqlFilter)) {
            final PropertyIgnoringFilterVisitor visitor = new PropertyIgnoringFilterVisitor(SCENE_ATTRIBUTES, type);
            actualFilter = (Filter) cqlFilter.accept(visitor, null);
        } else {
            actualFilter = cqlFilter;
        }
        final CqlFilterPredicate filterPredicate = new CqlFilterPredicate(actualFilter);
        iterator = Iterators.filter(iterator, filterPredicate);
    }
}

private ParsingResult indexMeasurements() {
    System.out.println("indexMeasurements()...");
    String message = "";
    try {
        files.openFile("csv", "r");
    } catch (IOException e) {
        e.printStackTrace();
        message += "[ERROR] Fail to open the csv file\n";
        return new ParsingResult(1, message);
    }
    Iterable<CSVRecord> records = null;
    try {
        records = CSVFormat.DEFAULT.withHeader().parse(files.getReader("csv"));
    } catch (IOException e) {
        e.printStackTrace();
        message += "[ERROR] Fail to parse header of the csv file\n";
        return new ParsingResult(1, message);
    }
    int total_count = 0;
    int batch_size = 10000;
    int nTimeStampCol = -1;
    int nTimeInstantCol = -1;
    int nIdCol = -1;
    for (MeasurementType mt : hadatacKb.getDataset().getMeasurementTypes()) {
        if (mt.getTimestampColumn() > -1) {
            nTimeStampCol = mt.getTimestampColumn();
        }
        if (mt.getTimeInstantColumn() > -1) {
            nTimeInstantCol = mt.getTimeInstantColumn();
        }
        if (mt.getIdColumn() > -1) {
            nIdCol = mt.getIdColumn();
        }
    }
    boolean isSubjectPlatform = Subject.isPlatform(hadatacKb.getDeployment().getPlatform().getUri());
    SolrClient solr = new HttpSolrClient(Play.application().configuration().getString("hadatac.solr.data") + Collections.DATA_ACQUISITION);
    for (CSVRecord record : records) {
        Iterator<MeasurementType> iter = hadatacKb.getDataset().getMeasurementTypes().iterator();
        while (iter.hasNext()) {
            MeasurementType measurementType = iter.next();
            if (measurementType.getTimestampColumn() > -1) {
                continue;
            }
            if (measurementType.getTimeInstantColumn() > -1) {
                continue;
            }
            if (measurementType.getIdColumn() > -1) {
                continue;
            }
            Measurement measurement = new Measurement();
            if (record.get(measurementType.getValueColumn() - 1).isEmpty()) {
                continue;
            } else {
                String originalValue = record.get(measurementType.getValueColumn() - 1);
                String codeValue = Subject.findCodeValue(measurementType.getCharacteristicUri(), originalValue);
                if (null == codeValue) {
                    measurement.setValue(originalValue);
                } else {
                    measurement.setValue(codeValue);
                }
            }
            if (nTimeStampCol > -1) {
                String sTime = record.get(nTimeStampCol - 1);
                int timeStamp = new BigDecimal(sTime).intValue();
                Date time = new Date((long) timeStamp * 1000);
                measurement.setTimestamp(time.toString());
            } else if (nTimeInstantCol > -1) {
                measurement.setTimestamp(record.get(nTimeInstantCol - 1));
            } else {
                measurement.setTimestamp("");
            }
            measurement.setStudyUri(ValueCellProcessing.replaceNameSpaceEx(hadatacKb.getDataAcquisition().getStudyUri()));
            if (nIdCol > -1) {
                if (measurementType.getEntityUri().equals(ValueCellProcessing.replacePrefixEx("sio:Human"))) {
                    Subject subject = Subject.findSubject(measurement.getStudyUri(), record.get(nIdCol - 1));
                    if (null != subject) {
                        String subjectUri = subject.getUri();
                        subjectUri = Subject.checkObjectUri(subjectUri, measurementType.getCharacteristicUri());
                        measurement.setObjectUri(subjectUri);
                    } else {
                        measurement.setObjectUri("");
                    }
                } else if (measurementType.getEntityUri().equals(ValueCellProcessing.replacePrefixEx("sio:Sample"))) {
                    String sampleUri = Subject.findSampleUri(measurement.getStudyUri(), record.get(nIdCol - 1));
                    if (sampleUri != null) {
                        measurement.setObjectUri(sampleUri);
                    } else {
                        measurement.setObjectUri("");
                    }
                }
            } else {
                if (isSubjectPlatform) {
                    measurement.setObjectUri(hadatacKb.getDeployment().getPlatform().getUri());
                } else {
                    measurement.setObjectUri("");
                }
            }
            measurement.setUri(ValueCellProcessing.replacePrefixEx(measurement.getStudyUri()) + "/" + ValueCellProcessing.replaceNameSpaceEx(hadatacKb.getDataAcquisition().getUri()).split(":")[1] + "/" + hadatacCcsv.getDataset().getLocalName() + "/" + measurementType.getLocalName() + "-" + total_count);
            measurement.setOwnerUri(hadatacKb.getDataAcquisition().getOwnerUri());
            measurement.setAcquisitionUri(hadatacKb.getDataAcquisition().getUri());
            measurement.setUnit(measurementType.getUnitLabel());
            measurement.setUnitUri(measurementType.getUnitUri());
            measurement.setCharacteristic(measurementType.getCharacteristicLabel());
            measurement.setCharacteristicUri(measurementType.getCharacteristicUri());
            measurement.setInstrumentModel(hadatacKb.getDeployment().getInstrument().getLabel());
            measurement.setInstrumentUri(hadatacKb.getDeployment().getInstrument().getUri());
            measurement.setPlatformName(hadatacKb.getDeployment().getPlatform().getLabel());
            measurement.setPlatformUri(hadatacKb.getDeployment().getPlatform().getUri());
            measurement.setEntity(measurementType.getEntityLabel());
            measurement.setEntityUri(measurementType.getEntityUri());
            measurement.setDatasetUri(hadatacCcsv.getDatasetKbUri());
            try {
                solr.addBean(measurement);
            } catch (IOExceptionSolrServerException |  e) {
                System.out.println("[ERROR] SolrClient.addBean - e.Message: " + e.getMessage());
            }
            if ((++total_count) % batch_size == 0) {
                try {
                    System.out.println("solr.commit()...");
                    solr.commit();
                    System.out.println(String.format("Committed %s measurements!", batch_size));
                } catch (IOExceptionSolrServerException |  e) {
                    System.out.println("[ERROR] SolrClient.commit - e.Message: " + e.getMessage());
                    message += "[ERROR] Fail to commit to solr\n";
                    try {
                        solr.close();
                    } catch (IOException e1) {
                        System.out.println("[ERROR] SolrClient.close - e.Message: " + e1.getMessage());
                        message += "[ERROR] Fail to close solr\n";
                    }
                    return new ParsingResult(1, message);
                }
            }
        }
    }
    try {
        try {
            System.out.println("solr.commit()...");
            solr.commit();
            System.out.println(String.format("Committed %s measurements!", total_count % batch_size));
        } catch (IOExceptionSolrServerException |  e) {
            solr.close();
            System.out.println("[ERROR] SolrClient.commit - e.Message: " + e.getMessage());
            message += "[ERROR] Fail to commit to solr\n";
            return new ParsingResult(1, message);
        }
        files.closeFile("csv", "r");
    } catch (IOException e) {
        e.printStackTrace();
        message += "[ERROR] Fail to close the csv file\n";
        return new ParsingResult(1, message);
    }
    hadatacKb.getDataAcquisition().addNumberDataPoints(total_count);
    hadatacKb.getDataAcquisition().save();
    System.out.println("Finished indexMeasurements()");
    try {
        solr.close();
    } catch (IOException e) {
        System.out.println("[ERROR] SolrClient.close - e.Message: " + e.getMessage());
        message += "[ERROR] Fail to close solr\n";
    }
    return new ParsingResult(0, message);
}

public List<IStoreItem> parse(String filePath) throws IOException {
    final List<IStoreItem> res = new ArrayList<IStoreItem>();
    final File inFile = new File(filePath);
    final Reader in = new InputStreamReader(new FileInputStream(inFile), Charset.forName("UTF-8"));
    final String fullFileName = inFile.getName();
    final String fileName = filePrefix(fullFileName);
    final Iterable<CSVRecord> records = CSVFormat.DEFAULT.parse(in);
    boolean first = true;
    // generate our list of importers
    createImporters();
    final DataImporter temporalDimensionless = new TemporalSeriesSupporter<Temporal.DimensionlessDouble>(Temporal.DimensionlessDouble.class, null, null);
    final DataImporter temporalStrings = new TemporalStringImporter();
    final DataImporter strings = new StringImporter();
    final DataImporter dimensionless = new SeriesSupporter<NonTemporal.DimensionlessDouble>(NonTemporal.DimensionlessDouble.class, null, null);
    // store one importer per column-set
    List<DataImporter> importers = new ArrayList<DataImporter>();
    // and store one series per column-set
    List<ICollection> series = new ArrayList<ICollection>();
    boolean isTime = false;
    DateFormat customDateFormat = null;
    for (CSVRecord record : records) {
        if (first) {
            first = false;
            String time = record.get(0);
            int ctr = 0;
            if (time != null && time.toLowerCase().startsWith("time")) {
                // is it plain time?
                if (!time.toLowerCase().equals("time")) {
                    // ok, see if we have a time format string
                    if (time.contains("(") && time.contains(")")) {
                        // ok, extract the format string
                        String formatStr = time.substring(time.indexOf("(") + 1, time.indexOf(")"));
                        customDateFormat = new SimpleDateFormat(formatStr);
                    }
                }
                isTime = true;
                ctr = 1;
            } else {
                ctr = 0;
            }
            while (ctr < record.size()) {
                String nextVal = record.get(ctr);
                // have a look at it.
                int i1 = nextVal.indexOf("(");
                String colName;
                if (i1 > 0) {
                    // ok, we have units
                    colName = nextVal.substring(0, i1).trim();
                } else {
                    // no, no units
                    colName = nextVal.trim();
                }
                // see if anybody can handle this name
                boolean handled = false;
                Iterator<DataImporter> cIter = _candidates.iterator();
                while (cIter.hasNext()) {
                    DataImporter thisI = cIter.next();
                    if (thisI.handleName(colName)) {
                        importers.add(thisI);
                        series.add(thisI.create(fileName + "-" + thisI.nameFor(colName)));
                        handled = true;
                        ctr += thisI.numCols();
                        break;
                    }
                }
                if (!handled) {
                    int i2 = nextVal.indexOf(")");
                    if (i2 > 0 && i2 > i1 + 1) {
                        final String units = nextVal.substring(i1 + 1, i2).trim();
                        Iterator<DataImporter> cIter2 = _candidates.iterator();
                        while (cIter2.hasNext()) {
                            DataImporter thisI = cIter2.next();
                            if (thisI.handleUnits(units)) {
                                importers.add(thisI);
                                series.add(thisI.create(fileName + "-" + thisI.nameFor(colName)));
                                ctr += thisI.numCols();
                                handled = true;
                                break;
                            }
                        }
                    }
                }
                // have we managed it?
                if (!handled) {
                    // ok, in that case we don't know. Let's introduce a deferred
                    // decision
                    // maker, so we can make a decision once we've read in some data
                    importers.add(new DeferredLoadSupporter(colName));
                    series.add(new ObjectCollection<String>("null"));
                    ctr += 1;
                }
            }
        } else {
            String firstRow = record.get(0);
            long theTime = -1;
            int thisCol = 0;
            // ok, we're out of the first row
            if (isTime) {
                // ok, get the time field
                try {
                    // do we have a custom date format
                    final DateFormat thisFormat;
                    if (customDateFormat != null) {
                        thisFormat = customDateFormat;
                    } else {
                        int len = firstRow.length();
                        if (len < 10) {
                            thisFormat = TIME_FORMAT;
                        } else {
                            thisFormat = DATE_FORMAT;
                        }
                    }
                    Date date = thisFormat.parse(firstRow);
                    theTime = date.getTime();
                    thisCol = 1;
                } catch (ParseException e) {
                    e.printStackTrace();
                }
            } else {
                // not temporal, use this field
                thisCol = 0;
            }
            // now move through the other cols
            int numImporters = importers.size();
            for (int i = 0; i < numImporters; i++) {
                DataImporter thisI = importers.get(i);
                // ok, just check if this is a deferred importer
                if (thisI instanceof DeferredLoadSupporter) {
                    DeferredLoadSupporter dl = (DeferredLoadSupporter) thisI;
                    String seriesName = dl.getName();
                    // ok, have a look at the next field
                    String nextVal = record.get(thisCol);
                    // is it numeric?
                    DataImporter importer = null;
                    // ok, treat it as string data
                    if (isTime) {
                        if (isNumeric(nextVal)) {
                            // ok, we've got dimensionless quantity data
                            importer = temporalDimensionless;
                        } else {
                            importer = temporalStrings;
                        }
                    } else {
                        if (isNumeric(nextVal)) {
                            // ok, we've got dimensionless quantity data
                            importer = dimensionless;
                        } else {
                            importer = strings;
                        }
                    }
                    if (importer != null) {
                        int index = importers.indexOf(dl);
                        importers.set(index, importer);
                        series.set(index, importer.create(fileName + "-" + seriesName));
                        thisI = importer;
                    }
                }
                ICollection thisS = series.get(i);
                thisI.consume(thisS, theTime, thisCol, record);
                thisCol += thisI.numCols();
            }
        }
    }
    // ok, store the series
    if (series.size() > 1) {
        StoreGroup target = new StoreGroup(fullFileName);
        Iterator<ICollection> sIter = series.iterator();
        while (sIter.hasNext()) {
            ICollection coll = (ICollection) sIter.next();
            target.add(coll);
        }
        res.add(target);
    } else {
        Iterator<ICollection> sIter = series.iterator();
        while (sIter.hasNext()) {
            ICollection coll = (ICollection) sIter.next();
            res.add(coll);
        }
    }
    return res;
}

/**
	 * Parse an inputstream and populate a Metadata object
	 * @param pInputStream stream to analyse 
	 * @param pMetadata metadata object to populate
	 * @param pOutputStream output to write data to
	 * @return true if processed ok, false if execution was terminated
	 */
public boolean parse(final InputStream pInputStream, final Metadata pMetadata) {
    boolean ret = true;
    if (!gRunner.isRunning()) {
        gLogger.error("Tika-Server is not running");
        return false;
    }
    final String TIKA_PATH = "/meta";
    final String END_POINT = "http://" + TIKA_LOCAL_HOST + ":" + TIKA_SERVER_PORT;
    gLogger.trace("Server: " + END_POINT + TIKA_PATH);
    final String detectedType = pMetadata.get(Metadata.CONTENT_TYPE);
    FutureTask<Integer> task = new FutureTask<Integer>(new Callable<Integer>() {

        @Override
        public Integer call() throws Exception {
            gResponse = WebClient.create(END_POINT + TIKA_PATH).accept("text/csv").type(// give the parsers a hint
            detectedType).put(// protect the stream from being closed
            new CloseShieldInputStream(pInputStream));
            return null;
        }
    });
    Thread thread = new Thread(task);
    thread.start();
    try {
        task.get(TIMEOUT_SECS * 1000, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        gLogger.info("InterruptedException: " + e);
        ret = false;
        restart();
    } catch (ExecutionException e) {
        gLogger.info("ExecutionException: " + e);
        ret = false;
        restart();
    } catch (TimeoutException e) {
        gLogger.info("TimeoutException: " + e);
        ret = false;
        restart();
    }
    if (gResponse != null) {
        if (gResponse.getStatus() == Status.UNSUPPORTED_MEDIA_TYPE.getStatusCode()) {
            // the server may return HTTP 415 (unsupported) if it won't accept the mimetype
            // handle this issue here
            // add some text to the output
            // FIXME: maybe change mimetype for a more visible error?
            pMetadata.add("parseFailure415", "true");
            gLogger.error("Parse Failure: HTTP 415 (format unsupported for parsing)");
        } else {
            if (gResponse.getEntity() instanceof InputStream) {
                InputStream is = (InputStream) gResponse.getEntity();
                BufferedReader reader = new BufferedReader(new InputStreamReader(is));
                try {
                    Iterable<CSVRecord> records = CSVFormat.DEFAULT.parse(reader);
                    for (CSVRecord record : records) {
                        pMetadata.add(record.get(0), record.get(1));
                    }
                } catch (IOException e1) {
                    e1.printStackTrace();
                    ret = false;
                } finally {
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        }
    }
    gLogger.info("Metadata entries: " + pMetadata.names().length);
    return ret;
}

/**
   * Produces a CSV response with results or facets.
   * 
   * @param findRequest
   *          the request parameters.
   * @param user
   *          the current {@link User}.
   * @param <T>
   *          Type of the resources to return.
   * @return a {@link Response} with CSV.
   * @throws RequestNotValidException
   *           it the request is not valid.
   * @throws AuthorizationDeniedException
   *           if the user is not authorized to perform this operation.
   * @throws GenericException
   *           if some other error occurs.
   */
private <T extends IsIndexed> Response csvResponse(final FindRequest findRequest, final User user) throws RequestNotValidException, AuthorizationDeniedException, GenericException {
    final Class<T> returnClass = getClass(findRequest.classToReturn);
    final Configuration config = RodaCoreFactory.getRodaConfiguration();
    final char delimiter;
    if (StringUtils.isBlank(config.getString(CONFIG_KEY_CSV_DELIMITER))) {
        delimiter = CSVFormat.DEFAULT.getDelimiter();
    } else {
        delimiter = config.getString(CONFIG_KEY_CSV_DELIMITER).trim().charAt(0);
    }
    if (findRequest.exportFacets) {
        final IndexResult<T> result = Browser.find(returnClass, findRequest.filter, Sorter.NONE, Sublist.NONE, findRequest.facets, user, findRequest.onlyActive, new ArrayList<>());
        return ApiUtils.okResponse(new RodaStreamingOutput(new FacetsCSVOutputStream(result.getFacetResults(), findRequest.filename, delimiter)).toStreamResponse());
    } else {
        final IterableIndexResult<T> result = Browser.findAll(returnClass, findRequest.filter, findRequest.sorter, findRequest.sublist, findRequest.facets, user, findRequest.onlyActive, new ArrayList<>());
        return ApiUtils.okResponse(new RodaStreamingOutput(new ResultsCSVOutputStream<>(result, findRequest.filename, delimiter)).toStreamResponse());
    }
}

/**
     *
     * @return
     */
private Iterable<CSVRecord> getCsv() {
    // we parse the csv file to extract the first line and get the headers 
    LineIterator lineIterator;
    try {
        lineIterator = FileUtils.lineIterator(dataFile, Charset.defaultCharset().name());
    } catch (IOException ex) {
        Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
        lineIterator = null;
    }
    String[] csvHeaders = lineIterator.next().split(String.valueOf(delimiter));
    isCriterionPresent = extractCriterionFromCsvHeader(csvHeaders);
    try {
        extractAvailableLangsFromCsvHeader(csvHeaders);
    } catch (I18NLanguageNotFoundException ex) {
        Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
        return null;
    }
    // from here we just add each line to a build to re-create the csv content
    // without the first line.
    StringBuilder strb = new StringBuilder();
    while (lineIterator.hasNext()) {
        strb.append(lineIterator.next());
        strb.append("\n");
    }
    Reader in;
    try {
        in = new StringReader(strb.toString());
        CSVFormat csvf = CSVFormat.newFormat(delimiter).withHeader(csvHeaders);
        return csvf.parse(in);
    } catch (FileNotFoundException ex) {
        Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
        return null;
    } catch (IOException ex) {
        Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
        return null;
    }
}

public static List<InstrumentText> buildList(Connection con, String strategy, LocalDate date, String csvPath, char csvSep) throws Exception {
    // public static List<InstrumentText> buildList(Connection con, String strategy, LocalDate date) throws Exception {
    ArrayList<InstrumentText> result = new ArrayList<InstrumentText>();
    CSVPrinter printer = null;
    if (csvPath != null) {
        // Add withHeader for headers
        printer = CSVFormat.DEFAULT.withDelimiter(csvSep).print(new BufferedWriter(new FileWriter(csvPath)));
    }
    int numCsvColumns = 12;
    int rollMethod = 2;
    DatabaseMetaData dmd = con.getMetaData();
    String driverName = dmd.getDriverName();
    String query = "";
    if (driverName.startsWith("MySQL")) {
        query = STRATEGY_QUERY_MYSQL;
    } else {
        query = STRATEGY_QUERY;
    }
    String prevCategory = "";
    PreparedStatement pstmt = con.prepareStatement(query);
    pstmt.setString(1, strategy);
    pstmt.setTimestamp(2, Timestamp.valueOf(date.atStartOfDay()));
    ResultSet rs = pstmt.executeQuery();
    while (rs.next()) {
        String category = rs.getString(2);
        if (!category.equals(prevCategory)) {
            result.add(InstrumentText.makeSection(category));
            prevCategory = category;
            if (printer != null) {
                printer.print(category);
                for (int ii = 1; ii < numCsvColumns; ++ii) {
                    printer.print("");
                }
                printer.println();
            }
        }
        String name = rs.getString(3);
        String symbol = rs.getString(4);
        String contract = "";
        if (rollMethod == 1) {
            // Uses current_contract and trading_days
            int ndays = rs.getInt(12);
            if (ndays > 1) {
                contract = rs.getString(10);
            } else {
                contract = "Roll to " + rs.getString(11);
            }
        } else if (rollMethod == 2) {
            // Uses current_contract2 and roll_today
            int rollToday = rs.getInt(14);
            if (rollToday == 0) {
                contract = rs.getString(13);
            } else {
                contract = "Roll to " + rs.getString(13);
            }
        }
        if (printer != null) {
            printer.print(name);
            printer.print(symbol);
            printer.print(contract);
        }
        String signal;
        long position = (long) rs.getDouble(5);
        JsonObject jo = new Gson().fromJson(rs.getString(9), JsonObject.class);
        if (position > 0.0) {
            BigDecimal entryPrice;
            double pnl;
            try {
                entryPrice = jo.get("entry_price").getAsBigDecimal();
                pnl = jo.get("pnl").getAsDouble();
            } catch (Exception e) {
                entryPrice = BigDecimal.valueOf(Double.MIN_VALUE);
                pnl = Double.MIN_VALUE;
            }
            signal = String.format("Long [%d] since %s [at %s].", position, rs.getString(6), formatBigDecimal(entryPrice));
            if (printer != null)
                printer.print(signal);
            String openProfit = String.format("Open equity profit %,d.", (int) Math.floor(pnl));
            signal += " " + openProfit;
            if (printer != null)
                printer.print(openProfit);
        } else if (position < 0.0) {
            BigDecimal entryPrice;
            double pnl;
            try {
                entryPrice = jo.get("entry_price").getAsBigDecimal();
                pnl = jo.get("pnl").getAsDouble();
            } catch (Exception e) {
                entryPrice = BigDecimal.valueOf(-1);
                pnl = -1;
            }
            signal = String.format("Short [%d] since %s [at %s].", Math.abs(position), rs.getString(6), formatBigDecimal(entryPrice));
            if (printer != null)
                printer.print(signal);
            String openProfit = String.format("Open equity profit %,d.", (int) Math.floor(pnl));
            signal += " " + openProfit;
            if (printer != null)
                printer.print(openProfit);
        } else {
            signal = "Out.";
            if (printer != null) {
                printer.print(signal);
                // An empty column follows the status if there is no position - there is no profit.
                printer.print("");
            }
        }
        boolean hasOrder = false;
        JsonArray ja = jo.get("orders").getAsJsonArray();
        double entryRisk;
        try {
            entryRisk = jo.get("entry_risk").getAsDouble();
        } catch (Exception ee) {
            entryRisk = Double.NaN;
        }
        String profitTarget;
        Double profitTargetDbl;
        try {
            profitTarget = formatBigDecimal(jo.get("profit_target").getAsBigDecimal());
            profitTargetDbl = jo.get("profit_target").getAsDouble();
        } catch (Exception ee) {
            profitTarget = null;
            profitTargetDbl = null;
        }
        String stopLoss;
        Double stopLossDbl;
        try {
            stopLoss = formatBigDecimal(jo.get("stop_loss").getAsBigDecimal());
            stopLossDbl = jo.get("stop_loss").getAsDouble();
        } catch (Exception ee) {
            stopLoss = null;
            stopLossDbl = null;
        }
        Double lastClose;
        try {
            lastClose = jo.get("last_close").getAsDouble();
        } catch (Exception ee) {
            lastClose = null;
        }
        // Currently maximum one entry and maximum one exit are supported.
        String entryStr = "";
        String exitStr = "";
        String contractRiskStr = "";
        for (int ii = 0; ii < ja.size(); ++ii) {
            JsonObject jorder = ja.get(ii).getAsJsonObject();
            switch(jorder.get("type").getAsString()) {
                case "EXIT_LONG_STOP":
                    exitStr = "Exit long at stop " + formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()) + ".";
                    signal += " " + exitStr;
                    break;
                case "EXIT_SHORT_STOP":
                    exitStr = "Exit short at stop " + formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()) + ".";
                    signal += " " + exitStr;
                    break;
                case "ENTER_LONG":
                    if (!Double.isNaN(entryRisk)) {
                        entryStr = String.format("Enter long at open. Contract risk is %s.", formatDouble(entryRisk, 0, 0));
                        signal += " " + entryStr;
                    } else {
                        entryStr = "Enter long at open.";
                        signal += " " + entryStr;
                    }
                    break;
                case "ENTER_SHORT":
                    if (!Double.isNaN(entryRisk)) {
                        entryStr = String.format("Enter short at open. Contract risk is %s.", formatDouble(entryRisk, 0, 0));
                        signal += " " + entryStr;
                    } else {
                        entryStr = "Enter short at open.";
                        signal += " " + entryStr;
                    }
                    break;
                case "ENTER_LONG_STOP":
                    position = jorder.get("quantity").getAsLong();
                    entryStr = String.format("Enter long [%d] at stop %s [%s%%].", position, formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()), formatPercentage(jorder.get("stop_price").getAsDouble() / lastClose * 100 - 100));
                    signal += " " + entryStr;
                    if (!Double.isNaN(entryRisk)) {
                        contractRiskStr = String.format(" Contract risk is %s.", formatDouble(entryRisk, 0, 0));
                        signal += " " + contractRiskStr;
                    }
                    break;
                case "ENTER_LONG_STOP_LIMIT":
                    position = jorder.get("quantity").getAsLong();
                    entryStr = String.format("Enter long [%d] at limit %s, stop at %s [%s%%].", position, formatBigDecimal(jorder.get("limit_price").getAsBigDecimal()), formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()), formatPercentage(jorder.get("stop_price").getAsDouble() / lastClose * 100 - 100));
                    signal += " " + entryStr;
                    if (!Double.isNaN(entryRisk)) {
                        contractRiskStr = String.format(" Contract risk is %s.", formatDouble(entryRisk, 0, 0));
                        signal += contractRiskStr;
                    }
                    break;
                case "ENTER_SHORT_STOP":
                    // signal += " Enter short at stop " + formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()) + ".";
                    position = jorder.get("quantity").getAsLong();
                    entryStr = String.format("Enter short [%d] at stop %s [%s%%].", Math.abs(position), formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()), formatPercentage(jorder.get("stop_price").getAsDouble() / lastClose * 100 - 100));
                    signal += " " + entryStr;
                    if (!Double.isNaN(entryRisk)) {
                        contractRiskStr = String.format(" Contract risk is %s.", formatDouble(entryRisk, 0, 0));
                        signal += " " + contractRiskStr;
                    }
                    break;
                case "ENTER_SHORT_STOP_LIMIT":
                    position = jorder.get("quantity").getAsLong();
                    entryStr = String.format("Enter short [%d] at limit %s, stop at %s [%s%%].", Math.abs(position), formatBigDecimal(jorder.get("limit_price").getAsBigDecimal()), formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()), formatPercentage(jorder.get("stop_price").getAsDouble() / lastClose * 100 - 100));
                    signal += " " + entryStr;
                    if (!Double.isNaN(entryRisk)) {
                        contractRiskStr = String.format(" Contract risk is %s.", formatDouble(entryRisk, 0, 0));
                        signal += " " + contractRiskStr;
                    }
                    break;
                case "EXIT_LONG":
                    exitStr = "Exit long at open.";
                    signal += " " + exitStr;
                    break;
                case "EXIT_SHORT":
                    exitStr = "Exit short at open.";
                    signal += " " + exitStr;
                    break;
                case "EXIT_SHORT_STOP_LIMIT":
                    exitStr = "Exit short at limit " + formatBigDecimal(jorder.get("limit_price").getAsBigDecimal()) + ", stop at " + formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()) + " [" + formatPercentage(jorder.get("stop_price").getAsDouble() / lastClose * 100 - 100) + "%]" + ".";
                    signal += " " + exitStr;
                    break;
                case "EXIT_LONG_STOP_LIMIT":
                    exitStr = "Exit long at limit " + formatBigDecimal(jorder.get("limit_price").getAsBigDecimal()) + ", stop at " + formatBigDecimal(jorder.get("stop_price").getAsBigDecimal()) + " [" + formatPercentage(jorder.get("stop_price").getAsDouble() / lastClose * 100 - 100) + "%]" + ".";
                    signal += " " + exitStr;
                    break;
            }
            hasOrder = true;
        }
        String lastCloseStr = "Last close at " + formatBigDecimal(jo.get("last_close").getAsBigDecimal()) + ".";
        String stopLossStr = "";
        String profitTargetStr = "";
        if (hasOrder) {
            signal += " " + lastCloseStr;
        }
        if (stopLoss != null) {
            stopLossStr = "Stop loss at " + stopLoss;
            if (lastClose != null && stopLossDbl != null) {
                stopLossStr += " [" + formatPercentage(stopLossDbl / lastClose * 100 - 100) + "%]";
            }
            stopLossStr += ".";
            signal += " " + stopLossStr;
        }
        if (profitTarget != null) {
            profitTargetStr = "Profit target at about " + profitTarget;
            if (profitTargetDbl != null && lastClose != null) {
                profitTargetStr += " [" + formatPercentage(profitTargetDbl / lastClose * 100 - 100) + "%]";
            }
            profitTargetStr += ".";
            signal += " " + profitTargetStr;
        }
        if (printer != null) {
            printer.print(exitStr);
            printer.print(entryStr);
            printer.print(contractRiskStr);
            printer.print(lastCloseStr);
            printer.print(stopLossStr);
            printer.print(profitTargetStr);
            printer.println();
        }
        result.add(InstrumentText.make(name, symbol, contract, signal));
    }
    rs.close();
    pstmt.close();
    if (printer != null)
        printer.flush();
    return result;
}

private void writeToCSVFile(FreeFlowData freeFlowData) {
    if (csvFileName == null)
        return;
    final File csvFile = new File(csvFileName);
    final boolean fileExists = csvFile.exists() == true || csvFile.length() > 0;
    lock.lock();
    if (!fileExists) {
        try {
            if (csvFilePrinter != null)
                csvFilePrinter.close();
            if (fileWriter != null)
                fileWriter.close();
        } catch (IOException e) {
            LOG.warn(e.getMessage(), e);
        }
        csvFilePrinter = null;
        fileWriter = null;
    }
    try {
        if (fileWriter == null) {
            try {
                fileWriter = new FileWriter(csvFile, true);
                if (csvFilePrinter == null) {
                    try {
                        csvFilePrinter = new CSVPrinter(fileWriter, CSVFormat.RFC4180.withCommentMarker('#'));
                    } catch (IOException e) {
                        csvFilePrinter = null;
                        LOG.error(e.getMessage(), e);
                        LOG.error("Failed to write line to CSV file: " + freeFlowData);
                        return;
                    }
                }
            } catch (IOException e) {
                fileWriter = null;
                LOG.error(e.getMessage(), e);
                LOG.error("Failed to write line to CSV file: " + freeFlowData);
                return;
            }
        }
        if (!fileExists) {
            LOG.info("Created new CSV file: " + csvFile.getAbsolutePath());
            csvFilePrinter.printComment("Generated by AIS Abnormal Behaviour Analyzer");
            csvFilePrinter.printComment("File created: " + LocalDateTime.now().format(fmt));
            csvFilePrinter.printRecord("TIMESTAMP (GMT)", "MMSI1", "NAME1", "TP1", "LOA1", "BM1", "COG1", "HDG1", "SOG1", "LAT1", "LON1", "MMSI2", "NAME2", "TP2", "LOA2", "BM2", "COG2", "HDG2", "SOG2", "LAT2", "LON2", "BRG", "DST");
        }
        final Track t0 = freeFlowData.getTrackSnapshot();
        final Position p0 = freeFlowData.getTrackCenterPosition();
        List<FreeFlowData.TrackInsideEllipse> tracks = freeFlowData.getTracksInsideEllipse();
        for (FreeFlowData.TrackInsideEllipse track : tracks) {
            final Track t1 = track.getTrackSnapshot();
            final Position p1 = track.getTrackCenterPosition();
            final int d = (int) p0.distanceTo(p1, CoordinateSystem.CARTESIAN);
            final int b = (int) p0.rhumbLineBearingTo(p1);
            List csvRecord = new ArrayList<>();
            csvRecord.add(String.format(Locale.ENGLISH, "%s", t0.getTimeOfLastPositionReportTyped().format(fmt)));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t0.getMmsi()));
            csvRecord.add(String.format(Locale.ENGLISH, "%s", trimAisString(t0.getShipName()).replace(',', ' ')));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t0.getShipType()));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t0.getVesselLength()));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t0.getVesselBeam()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.0f", t0.getCourseOverGround()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.0f", t0.getTrueHeading()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.0f", t0.getSpeedOverGround()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.4f", p0.getLatitude()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.4f", p0.getLongitude()));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t1.getMmsi()));
            csvRecord.add(String.format(Locale.ENGLISH, "%s", trimAisString(t1.getShipName()).replace(',', ' ')));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t1.getShipType()));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t1.getVesselLength()));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", t1.getVesselBeam()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.0f", t1.getCourseOverGround()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.0f", t1.getTrueHeading()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.0f", t1.getSpeedOverGround()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.4f", p1.getLatitude()));
            csvRecord.add(String.format(Locale.ENGLISH, "%.4f", p1.getLongitude()));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", b));
            csvRecord.add(String.format(Locale.ENGLISH, "%d", d));
            try {
                csvFilePrinter.printRecord(csvRecord);
            } catch (IOException e) {
                LOG.error(e.getMessage(), e);
                LOG.error("Failed to write line to CSV file: " + freeFlowData);
            }
        }
        csvFilePrinter.flush();
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
    } finally {
        lock.unlock();
    }
}

private void releaseLock() throws CloudsyncException {
    if (!isLocked || duplicates.size() > 0)
        return;
    try {
        Files.delete(lockFilePath);
    } catch (IOException e) {
        throw new CloudsyncException("Couldn't remove '" + lockFilePath.toString() + "'");
    }
    try {
        if (root.getChildren().size() > 0) {
            LOGGER.log(Level.INFO, "write structure to cache file");
            final PrintWriter out = new PrintWriter(cacheFilePath.toFile());
            final CSVPrinter csvOut = new CSVPrinter(out, CSVFormat.EXCEL);
            writeStructureToCSVPrinter(csvOut, root);
            out.close();
        }
    } catch (final IOException e) {
        throw new CloudsyncException("Can't write cache file on '" + cacheFilePath.toString() + "'", e);
    }
    isLocked = false;
}

private void iterateOverZipFile(List<byte[]> theZipBytes, String fileNamePart, IRecordHandler handler, char theDelimiter, QuoteMode theQuoteMode) {
    boolean found = false;
    for (byte[] nextZipBytes : theZipBytes) {
        ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new ByteArrayInputStream(nextZipBytes)));
        try {
            for (ZipEntry nextEntry; (nextEntry = zis.getNextEntry()) != null; ) {
                ZippedFileInputStream inputStream = new ZippedFileInputStream(zis);
                String nextFilename = nextEntry.getName();
                if (nextFilename.contains(fileNamePart)) {
                    ourLog.info("Processing file {}", nextFilename);
                    found = true;
                    Reader reader = null;
                    CSVParser parsed = null;
                    try {
                        reader = new InputStreamReader(new BOMInputStream(zis), Charsets.UTF_8);
                        CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
                        if (theQuoteMode != null) {
                            format = format.withQuote('"').withQuoteMode(theQuoteMode);
                        }
                        parsed = new CSVParser(reader, format);
                        Iterator<CSVRecord> iter = parsed.iterator();
                        ourLog.debug("Header map: {}", parsed.getHeaderMap());
                        int count = 0;
                        int logIncrement = LOG_INCREMENT;
                        int nextLoggedCount = 0;
                        while (iter.hasNext()) {
                            CSVRecord nextRecord = iter.next();
                            handler.accept(nextRecord);
                            count++;
                            if (count >= nextLoggedCount) {
                                ourLog.info(" * Processed {} records in {}", count, nextFilename);
                                nextLoggedCount += logIncrement;
                            }
                        }
                    } catch (IOException e) {
                        throw new InternalErrorException(e);
                    }
                }
            }
        } catch (IOException e) {
            throw new InternalErrorException(e);
        } finally {
            IOUtils.closeQuietly(zis);
        }
    }
    // This should always be true, but just in case we've introduced a bug...
    Validate.isTrue(found);
}

/**
     *  Write Result to CSV format file
     * @param table          table name
     * @param outputPath Local custom output path
     * @param  finalName      if outputPath contain file name
     * @return                   The local absolute path of the file output
     */
public String writeAsCSV(String table, String outputPath, boolean finalName) {
    // Remove the extra separator
    while (outputPath.lastIndexOf(File.separator) == (outputPath.length() - 1)) {
        outputPath = outputPath.substring(0, outputPath.length() - 1);
    }
    String resultPath = "";
    if (finalName)
        resultPath = outputPath + "@" + Instant.now().toEpochMilli() + ".csv";
    else
        resultPath = outputPath + File.separator + table + "@" + Instant.now().toEpochMilli() + ".csv";
    ResultMap resultMap = readJsonAsResult(table, 0, -1);
    File csvfile = new File(resultPath);
    // create parent dir
    if (!csvfile.getParentFile().exists())
        csvfile.getParentFile().mkdirs();
    if (csvfile.exists())
        csvfile.delete();
    CSVPrinter csvp = null;
    CSVFormat format = CSVFormat.EXCEL;
    try {
        FileWriter fw = new FileWriter(csvfile);
        csvp = new CSVPrinter(fw, format);
        csvp.printRecord(resultMap.getSchema());
        for (int i = 0; i < resultMap.getCount(); i++) {
            csvp.printRecord(resultMap.getData().get(i));
            if (i % batchSize == 0)
                fw.flush();
        }
        fw.flush();
        fw.close();
        csvp.close();
    } catch (IOException e) {
        this.setSuccess(false);
        this.setException(ExceptionUtils.getStackTrace(e));
        return "";
    }
    return resultPath;
}

/**
     * Classify unseen instances.
     * It builds the classifier against the training set
     * and then assign a classification to each unlabeled instance.
     *
     * @param output_path optional path to store a CSV file with the results.
     */
public void buildAndClassify(String output_path) {
    try {
        this.loadData(false);
    } catch (Exception e) {
        logger.fatal("Error while loading training and unlabeled data", e);
        return;
    }
    CSVPrinter csvFilePrinter = null;
    FileWriter fileWriter = null;
    if (output_path != null) {
        final CSVFormat csvFileFormat = CSVFormat.EXCEL.withDelimiter(CSV_DELIMITER);
        try {
            fileWriter = new FileWriter(output_path);
            csvFilePrinter = new CSVPrinter(fileWriter, csvFileFormat);
            csvFilePrinter.printRecord(CSV_FILE_HEADER);
        } catch (IOException e) {
            logger.warn("Error while creating CSV file printer", e);
            IOUtils.closeQuietly(fileWriter);
            IOUtils.closeQuietly(csvFilePrinter);
        }
    }
    try {
        logger.info("Building classifier {}...", this.classifier.getClass().getSimpleName());
        this.trainClassifier(this.training_data);
    } catch (Exception e) {
        logger.fatal("Error while building classifier for new instances.", e);
        IOUtils.closeQuietly(fileWriter);
        IOUtils.closeQuietly(csvFilePrinter);
        return;
    }
    final Attribute attribute_id = this.classification_data.attribute(Storage.ID);
    final Attribute attribute_lang = this.classification_data.attribute(Storage.LANG);
    final Attribute attribute_utc_offset = this.classification_data.attribute(Storage.UTC_OFFSET);
    final Attribute attribute_timezone = this.classification_data.attribute(Storage.TIMEZONE);
    Remove remove;
    try {
        remove = new Remove();
        remove.setAttributeIndices(String.format("%d", attribute_id.index() + 1));
        remove.setInputFormat(this.classification_data);
    } catch (Exception e) {
        logger.error("Error while building remove filter for unlabeled instances", e);
        return;
    }
    for (Instance i : this.classification_data) {
        remove.input(i);
        Instance trimmedInstance = remove.output();
        final long id = Double.valueOf(i.value(attribute_id)).longValue();
        double classification;
        try {
            classification = this.classifier.classifyInstance(trimmedInstance);
        } catch (Exception e) {
            logger.warn("Classification - id: {}, class: UNAVAILABLE", id);
            logger.error("Error while classifying unlabeled instance", e);
            return;
        }
        String location;
        if (this.wordsToKeep > 0) {
            StringBuilder locationBuilder = new StringBuilder();
            for (Attribute attribute : Collections.list(i.enumerateAttributes())) {
                if (attribute.name().startsWith(LOCATION_PREFIX) && i.value(attribute) > 0) {
                    locationBuilder.append(attribute.name().substring(LOCATION_PREFIX.length()));
                    locationBuilder.append(" ");
                }
            }
            location = locationBuilder.toString();
        } else {
            location = i.stringValue(this.classification_data.attribute(Storage.LOCATION));
        }
        final Object[] values = { id, String.format(Constants.twitter_user_intent, id), location, i.stringValue(attribute_lang), i.stringValue(attribute_utc_offset), i.stringValue(attribute_timezone), this.training_data.classAttribute().value((int) classification) };
        logger.debug("Classification - {} -> {}", i.toString(), values[6]);
        if (csvFilePrinter != null) {
            try {
                csvFilePrinter.printRecord(values);
            } catch (IOException e) {
                logger.error("Error while printing CSV records for ID {}", id, e);
            }
        }
    }
    IOUtils.closeQuietly(fileWriter);
    IOUtils.closeQuietly(csvFilePrinter);
}

public static void main(String[] args) throws Exception {
    Options opts = new Options();
    for (Option.Builder b : OPTION_BUILDERS) {
        opts.addOption(b.build());
    }
    CommandLine cl = null;
    try {
        CommandLineParser parser = new DefaultParser();
        cl = parser.parse(opts, args);
    } catch (ParseException e) {
        System.err.format("Argument parsing failed: %s\n", e.getMessage());
        HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        System.exit(1);
    }
    if (cl.hasOption("help")) {
        HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        return;
    }
    File lcmsDir = new File(cl.getOptionValue(OPTION_DIRECTORY));
    if (!lcmsDir.isDirectory()) {
        System.err.format("File at %s is not a directory\n", lcmsDir.getAbsolutePath());
        HELP_FORMATTER.printHelp(LoadPlateCompositionIntoDB.class.getCanonicalName(), HELP_MESSAGE, opts, null, true);
        System.exit(1);
    }
    try (DB db = DB.openDBFromCLI(cl)) {
        ScanFile.insertOrUpdateScanFilesInDirectory(db, lcmsDir);
        StandardIonAnalysis analysis = new StandardIonAnalysis();
        HashMap<Integer, Plate> plateCache = new HashMap<>();
        String plateBarcode = cl.getOptionValue(OPTION_STANDARD_PLATE_BARCODE);
        String inputChemicals = cl.getOptionValue(OPTION_STANDARD_CHEMICAL);
        String medium = cl.getOptionValue(OPTION_MEDIUM);
        // If standard chemical is specified, do standard LCMS ion selection analysis
        if (inputChemicals != null && !inputChemicals.equals("")) {
            String[] chemicals;
            if (!inputChemicals.contains(",")) {
                chemicals = new String[1];
                chemicals[0] = inputChemicals;
            } else {
                chemicals = inputChemicals.split(",");
            }
            String outAnalysis = cl.getOptionValue(OPTION_OUTPUT_PREFIX) + "." + CSV_FORMAT;
            String plottingDirectory = cl.getOptionValue(OPTION_PLOTTING_DIR);
            String[] headerStrings = { "Molecule", "Plate Bar Code", "LCMS Detection Results" };
            CSVPrinter printer = new CSVPrinter(new FileWriter(outAnalysis), CSVFormat.DEFAULT.withHeader(headerStrings));
            for (String inputChemical : chemicals) {
                List<StandardWell> standardWells;
                Plate queryPlate = Plate.getPlateByBarcode(db, cl.getOptionValue(OPTION_STANDARD_PLATE_BARCODE));
                if (plateBarcode != null && medium != null) {
                    standardWells = analysis.getStandardWellsForChemicalInSpecificPlateAndMedium(db, inputChemical, queryPlate.getId(), medium);
                } else if (plateBarcode != null) {
                    standardWells = analysis.getStandardWellsForChemicalInSpecificPlate(db, inputChemical, queryPlate.getId());
                } else {
                    standardWells = analysis.getStandardWellsForChemical(db, inputChemical);
                }
                if (standardWells.size() == 0) {
                    throw new RuntimeException("Found no LCMS wells for " + inputChemical);
                }
                // Sort in descending order of media where MeOH and Water related media are promoted to the top and
                // anything derived from yeast media are demoted.
                Collections.sort(standardWells, new Comparator<StandardWell>() {

                    @Override
                    public int compare(StandardWell o1, StandardWell o2) {
                        if (StandardWell.doesMediaContainYeastExtract(o1.getMedia()) && !StandardWell.doesMediaContainYeastExtract(o2.getMedia())) {
                            return 1;
                        } else {
                            return 0;
                        }
                    }
                });
                Map<StandardWell, StandardIonResult> wellToIonRanking = StandardIonAnalysis.getBestMetlinIonsForChemical(inputChemical, lcmsDir, db, standardWells, plottingDirectory);
                if (wellToIonRanking.size() != standardWells.size() && !cl.hasOption(OPTION_OVERRIDE_NO_SCAN_FILE_FOUND)) {
                    throw new Exception("Could not find a scan file associated with one of the standard wells");
                }
                for (StandardWell well : wellToIonRanking.keySet()) {
                    LinkedHashMap<String, XZ> snrResults = wellToIonRanking.get(well).getAnalysisResults();
                    String snrRankingResults = "";
                    int numResultsToShow = 0;
                    Plate plateForWellToAnalyze = Plate.getPlateById(db, well.getPlateId());
                    for (Map.Entry<String, XZ> ionToSnrAndTime : snrResults.entrySet()) {
                        if (numResultsToShow > 3) {
                            break;
                        }
                        String ion = ionToSnrAndTime.getKey();
                        XZ snrAndTime = ionToSnrAndTime.getValue();
                        snrRankingResults += String.format(ion + " (%.2f SNR at %.2fs); ", snrAndTime.getIntensity(), snrAndTime.getTime());
                        numResultsToShow++;
                    }
                    String[] resultSet = { inputChemical, plateForWellToAnalyze.getBarcode() + " " + well.getCoordinatesString() + " " + well.getMedia() + " " + well.getConcentration(), snrRankingResults };
                    printer.printRecord(resultSet);
                }
            }
            try {
                printer.flush();
                printer.close();
            } catch (IOException e) {
                System.err.println("Error while flushing/closing csv writer.");
                e.printStackTrace();
            }
        } else {
            // Get the set of chemicals that includes the construct and all it's intermediates
            Pair<ConstructEntry, List<ChemicalAssociatedWithPathway>> constructAndPathwayChems = analysis.getChemicalsForConstruct(db, cl.getOptionValue(OPTION_CONSTRUCT));
            System.out.format("Construct: %s\n", constructAndPathwayChems.getLeft().getCompositionId());
            for (ChemicalAssociatedWithPathway pathwayChem : constructAndPathwayChems.getRight()) {
                System.out.format("  Pathway chem %s\n", pathwayChem.getChemical());
                // Get all the standard wells for the pathway chemicals. These wells contain only the
                // the chemical added with controlled solutions (ie no organism or other chemicals in the
                // solution)
                List<StandardWell> standardWells;
                if (plateBarcode != null) {
                    Plate queryPlate = Plate.getPlateByBarcode(db, cl.getOptionValue(OPTION_STANDARD_PLATE_BARCODE));
                    standardWells = analysis.getStandardWellsForChemicalInSpecificPlate(db, pathwayChem.getChemical(), queryPlate.getId());
                } else {
                    standardWells = analysis.getStandardWellsForChemical(db, pathwayChem.getChemical());
                }
                for (StandardWell wellToAnalyze : standardWells) {
                    List<StandardWell> negativeControls = analysis.getViableNegativeControlsForStandardWell(db, wellToAnalyze);
                    Map<StandardWell, List<ScanFile>> allViableScanFiles = analysis.getViableScanFilesForStandardWells(db, wellToAnalyze, negativeControls);
                    List<String> primaryStandardScanFileNames = new ArrayList<>();
                    for (ScanFile scanFile : allViableScanFiles.get(wellToAnalyze)) {
                        primaryStandardScanFileNames.add(scanFile.getFilename());
                    }
                    Plate plate = plateCache.get(wellToAnalyze.getPlateId());
                    if (plate == null) {
                        plate = Plate.getPlateById(db, wellToAnalyze.getPlateId());
                        plateCache.put(plate.getId(), plate);
                    }
                    System.out.format("    Standard well: %s @ %s, '%s'%s%s\n", plate.getBarcode(), wellToAnalyze.getCoordinatesString(), wellToAnalyze.getChemical(), wellToAnalyze.getMedia() == null ? "" : String.format(" in %s", wellToAnalyze.getMedia()), wellToAnalyze.getConcentration() == null ? "" : String.format(" @ %s", wellToAnalyze.getConcentration()));
                    System.out.format("      Scan files: %s\n", StringUtils.join(primaryStandardScanFileNames, ", "));
                    for (StandardWell negCtrlWell : negativeControls) {
                        plate = plateCache.get(negCtrlWell.getPlateId());
                        if (plate == null) {
                            plate = Plate.getPlateById(db, negCtrlWell.getPlateId());
                            plateCache.put(plate.getId(), plate);
                        }
                        List<String> negativeControlScanFileNames = new ArrayList<>();
                        for (ScanFile scanFile : allViableScanFiles.get(negCtrlWell)) {
                            negativeControlScanFileNames.add(scanFile.getFilename());
                        }
                        System.out.format("      Viable negative: %s @ %s, '%s'%s%s\n", plate.getBarcode(), negCtrlWell.getCoordinatesString(), negCtrlWell.getChemical(), negCtrlWell.getMedia() == null ? "" : String.format(" in %s", negCtrlWell.getMedia()), negCtrlWell.getConcentration() == null ? "" : String.format(" @ %s", negCtrlWell.getConcentration()));
                        System.out.format("        Scan files: %s\n", StringUtils.join(negativeControlScanFileNames, ", "));
                    // TODO: do something useful with the standard wells and their scan files, and then stop all the printing.
                    }
                }
            }
        }
    }
}

/**
         * It builds a Dataframe object from a CSV file; the first line of the provided
         * CSV file must have a header with the column names.
         *
         * The method accepts the following arguments: A Reader object from where
         * we will read the contents of the csv file. The name column of the
         * response variable y. A map with the column names and their respective
         * DataTypes. The char delimiter for the columns, the char for quotes and
         * the string of the record/row separator. The Storage Configuration
         * object.
         *
         * @param reader
         * @param yVariable
         * @param headerDataTypes
         * @param delimiter
         * @param quote
         * @param recordSeparator
         * @param skip
         * @param limit
         * @param configuration
         * @return
         */
public static Dataframe parseCSVFile(Reader reader, String yVariable, LinkedHashMap<String, TypeInference.DataType> headerDataTypes, char delimiter, char quote, String recordSeparator, Long skip, Long limit, Configuration configuration) {
    Logger logger = LoggerFactory.getLogger(Dataframe.Builder.class);
    if (skip == null) {
        skip = 0L;
    }
    if (limit == null) {
        limit = Long.MAX_VALUE;
    }
    logger.info("Parsing CSV file");
    if (!headerDataTypes.containsKey(yVariable)) {
        logger.warn("WARNING: The file is missing the response variable column {}.", yVariable);
    }
    TypeInference.DataType yDataType = headerDataTypes.get(yVariable);
    //copy header types
    Map<String, TypeInference.DataType> xDataTypes = new HashMap<>(headerDataTypes);
    //remove the response variable from xDataTypes
    xDataTypes.remove(yVariable);
    //use the private constructor to pass DataTypes directly and avoid updating them on the fly
    Dataframe dataset = new Dataframe(configuration, yDataType, xDataTypes);
    CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(delimiter).withQuote(quote).withRecordSeparator(recordSeparator);
    try (final CSVParser parser = new CSVParser(reader, format)) {
        ThreadMethods.throttledExecution(StreamMethods.enumerate(StreamMethods.stream(parser.spliterator(), false)).skip(skip).limit(limit),  e -> {
            Integer rId = e.getKey();
            CSVRecord row = e.getValue();
            if (!row.isConsistent()) {
                logger.warn("WARNING: Skipping row {} because its size does not match the header size.", row.getRecordNumber());
            } else {
                Object y = null;
                AssociativeArray xData = new AssociativeArray();
                for (Map.Entry<String, TypeInference.DataType> entry : headerDataTypes.entrySet()) {
                    String column = entry.getKey();
                    TypeInference.DataType dataType = entry.getValue();
                    Object value = TypeInference.DataType.parse(row.get(column), dataType);
                    if (yVariable != null && yVariable.equals(column)) {
                        y = value;
                    } else {
                        xData.put(column, value);
                    }
                }
                Record r = new Record(xData, y);
                dataset._unsafe_set(rId, r);
            }
        }, configuration.getConcurrencyConfiguration());
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    return dataset;
}

public void writeTo(OutputStream outputStream) throws IOException {
    try (OutputStreamWriter osw = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)) {
        CSVPrinter csvPrinter = new CSVPrinter(osw, CSVFormat.DEFAULT);
        for (Object doc : docs) {
            if (doc instanceof Map) {
                Map<String, Object> map = (Map<String, Object>) doc;
                csvPrinter.printRecord(map.values());
            } else {
                csvPrinter.print(doc);
                csvPrinter.println();
            }
        }
        csvPrinter.flush();
        csvPrinter.close();
    }
}

public static Map<String, Object> importParty(DispatchContext dctx, Map<String, Object> context) {
    Delegator delegator = dctx.getDelegator();
    LocalDispatcher dispatcher = dctx.getDispatcher();
    Locale locale = (Locale) context.get("locale");
    GenericValue userLogin = (GenericValue) context.get("userLogin");
    ByteBuffer fileBytes = (ByteBuffer) context.get("uploadedFile");
    String encoding = System.getProperty("file.encoding");
    String csvString = Charset.forName(encoding).decode(fileBytes).toString();
    final BufferedReader csvReader = new BufferedReader(new StringReader(csvString));
    CSVFormat fmt = CSVFormat.DEFAULT.withHeader();
    List<String> errMsgs = new LinkedList<String>();
    List<String> newErrMsgs = new LinkedList<String>();
    // last partyId read from the csv file
    String lastPartyId = null;
    // current partyId from the csv file
    String currentPartyId = null;
    // new to create/update partyId in the system
    String newPartyId = null;
    String newCompanyPartyId = null;
    int partiesCreated = 0;
    Map<String, Object> result = null;
    String newContactMechId = null;
    String currentContactMechTypeId = null;
    String lastAddress1 = null;
    String lastAddress2 = null;
    String lastCity = null;
    String lastCountryGeoId = null;
    String lastEmailAddress = null;
    String lastCountryCode = null;
    String lastAreaCode = null;
    String lastContactNumber = null;
    String lastContactMechPurposeTypeId = null;
    String currentContactMechPurposeTypeId = null;
    // when modify party, contact mech not added again
    Boolean addParty = false;
    if (fileBytes == null) {
        return ServiceUtil.returnError(UtilProperties.getMessage(resourceError, "PartyUploadedFileDataNotFound", locale));
    }
    try {
        for (final CSVRecord rec : fmt.parse(csvReader)) {
            if (UtilValidate.isNotEmpty(rec.get("partyId"))) {
                currentPartyId = rec.get("partyId");
            }
            if (lastPartyId == null || !currentPartyId.equals(lastPartyId)) {
                newPartyId = null;
                currentContactMechPurposeTypeId = null;
                lastAddress1 = null;
                lastAddress2 = null;
                lastCity = null;
                lastCountryGeoId = null;
                lastEmailAddress = null;
                lastCountryCode = null;
                lastAreaCode = null;
                lastContactNumber = null;
                // party validation
                List<GenericValue> currencyCheck = EntityQuery.use(delegator).from("Uom").where("abbreviation", rec.get("preferredCurrencyUomId"), "uomTypeId", "CURRENCY_MEASURE").queryList();
                if (UtilValidate.isNotEmpty(rec.get("preferredCurrencyUomId")) && currencyCheck.size() == 0) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Currency code not found for: " + rec.get("preferredCurrencyUomId"));
                }
                if (UtilValidate.isEmpty(rec.get("roleTypeId"))) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory roletype is missing, possible values: CUSTOMER, SUPPLIER, EMPLOYEE and more....");
                } else if (EntityQuery.use(delegator).from("RoleType").where("roleTypeId", rec.get("roleTypeId")).queryOne() == null) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": RoletypeId is not valid: " + rec.get("roleTypeId"));
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && EntityQuery.use(delegator).from("ContactMechType").where("contactMechTypeId", rec.get("contactMechTypeId")).cache().queryOne() == null) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + " contactMechTypeId code not found for: " + rec.get("contactMechTypeId"));
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechPurposeTypeId")) && EntityQuery.use(delegator).from("ContactMechPurposeType").where("contactMechPurposeTypeId", rec.get("contactMechPurposeTypeId")).cache().queryOne() == null) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "contactMechPurposeTypeId code not found for: " + rec.get("contactMechPurposeTypeId"));
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "POSTAL_ADDRESS".equals(rec.get("contactMechTypeId"))) {
                    if (UtilValidate.isEmpty(rec.get("countryGeoId"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Country code missing");
                    } else {
                        List<GenericValue> countryCheck = EntityQuery.use(delegator).from("Geo").where("geoTypeId", "COUNTRY", "abbreviation", rec.get("countryGeoId")).queryList();
                        if (countryCheck.size() == 0) {
                            newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid Country code: " + rec.get("countryGeoId"));
                        }
                    }
                    if (UtilValidate.isEmpty(rec.get("city"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + "City name is missing");
                    }
                    if (UtilValidate.isNotEmpty(rec.get("stateProvinceGeoId"))) {
                        List<GenericValue> stateCheck = EntityQuery.use(delegator).from("Geo").where("geoTypeId", "STATE", "abbreviation", rec.get("stateProvinceGeoId")).queryList();
                        if (stateCheck.size() == 0) {
                            newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid stateProvinceGeoId code: " + rec.get("countryGeoId"));
                        }
                    }
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "TELECOM_NUMBER".equals(rec.get("contactMechTypeId"))) {
                    if (UtilValidate.isEmpty(rec.get("telAreaCode")) && UtilValidate.isEmpty(rec.get("telAreaCode"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " telephone number missing");
                    }
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "EMAIL_ADDRESS".equals(rec.get("contactMechTypeId"))) {
                    if (UtilValidate.isEmpty(rec.get("emailAddress"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " email address missing");
                    }
                }
                if (errMsgs.size() == 0) {
                    List<GenericValue> partyCheck = EntityQuery.use(delegator).from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")).queryList();
                    addParty = partyCheck.size() == 0;
                    if (!addParty) {
                        // update party
                        newPartyId = EntityUtil.getFirst(partyCheck).getString("partyId");
                        if (UtilValidate.isNotEmpty(rec.get("groupName"))) {
                            Map<String, Object> partyGroup = UtilMisc.toMap("partyId", newPartyId, "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin);
                            result = dispatcher.runSync("updatePartyGroup", partyGroup);
                        } else {
                            // person
                            Map<String, Object> person = UtilMisc.toMap("partyId", newPartyId, "firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "userLogin", userLogin);
                            result = dispatcher.runSync("updatePerson", person);
                        }
                    } else {
                        // create new party
                        if (UtilValidate.isNotEmpty(rec.get("groupName"))) {
                            Map<String, Object> partyGroup = UtilMisc.toMap("preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin, "statusId", "PARTY_ENABLED");
                            result = dispatcher.runSync("createPartyGroup", partyGroup);
                        } else {
                            // person
                            Map<String, Object> person = UtilMisc.toMap("firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "statusId", "PARTY_ENABLED", "userLogin", userLogin);
                            result = dispatcher.runSync("createPerson", person);
                        }
                        newPartyId = (String) result.get("partyId");
                        Map<String, Object> partyIdentification = UtilMisc.toMap("partyId", newPartyId, "partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId"), "userLogin", userLogin);
                        result = dispatcher.runSync("createPartyIdentification", partyIdentification);
                        Map<String, Object> partyRole = UtilMisc.toMap("partyId", newPartyId, "roleTypeId", rec.get("roleTypeId"), "userLogin", userLogin);
                        dispatcher.runSync("createPartyRole", partyRole);
                        if (UtilValidate.isNotEmpty(rec.get("companyPartyId"))) {
                            List<GenericValue> companyCheck = EntityQuery.use(delegator).from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")).queryList();
                            if (companyCheck.size() == 0) {
                                // update party group
                                // company does not exist so create
                                Map<String, Object> companyPartyGroup = UtilMisc.toMap("partyId", newCompanyPartyId, "statusId", "PARTY_ENABLED", "userLogin", userLogin);
                                result = dispatcher.runSync("createPartyGroup", companyPartyGroup);
                                newCompanyPartyId = (String) result.get("partyId");
                            } else {
                                newCompanyPartyId = EntityUtil.getFirst(companyCheck).getString("partyId");
                            }
                            Map<String, Object> companyRole = UtilMisc.toMap("partyId", newCompanyPartyId, "roleTypeId", "ACCOUNT", "userLogin", userLogin);
                            dispatcher.runSync("createPartyRole", companyRole);
                            // company exist, so create link
                            Map<String, Object> partyRelationship = UtilMisc.toMap("partyIdTo", newPartyId, "partyIdFrom", newCompanyPartyId, "roleTypeIdFrom", "ACCOUNT", "partyRelationshipTypeId", "EMPLOYMENT", "userLogin", userLogin);
                            result = dispatcher.runSync("createPartyRelationship", partyRelationship);
                        }
                    }
                    Debug.logInfo(" =========================================================party created id: " + newPartyId, module);
                    partiesCreated++;
                } else {
                    errMsgs.addAll(newErrMsgs);
                    newErrMsgs = new LinkedList<String>();
                }
            }
            currentContactMechTypeId = rec.get("contactMechTypeId");
            currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId");
            // party correctly created (not updated) and contactMechtype provided?
            if (newPartyId != null && addParty && UtilValidate.isNotEmpty(currentContactMechTypeId)) {
                // fill maps and check changes
                Map<String, Object> emailAddress = UtilMisc.toMap("contactMechTypeId", "EMAIL_ADDRESS", "userLogin", userLogin);
                Boolean emailAddressChanged = false;
                if ("EMAIL_ADDRESS".equals(currentContactMechTypeId)) {
                    emailAddress.put("infoString", rec.get("emailAddress"));
                    emailAddressChanged = lastEmailAddress == null || !lastEmailAddress.equals(rec.get("emailAddress"));
                    lastEmailAddress = rec.get("emailAddress");
                }
                // casting is here necessary for some compiler versions
                Map<String, Object> postalAddress = UtilMisc.toMap("userLogin", (Object) userLogin);
                Boolean postalAddressChanged = false;
                if ("POSTAL_ADDRESS".equals(currentContactMechTypeId)) {
                    postalAddress.put("address1", rec.get("address1"));
                    postalAddress.put("address2", rec.get("address2"));
                    postalAddress.put("city", rec.get("city"));
                    postalAddress.put("stateProvinceGeoId", rec.get("stateProvinceGeoId"));
                    postalAddress.put("countryGeoId", rec.get("countryGeoId"));
                    postalAddress.put("postalCode", rec.get("postalCode"));
                    postalAddressChanged = lastAddress1 == null || !lastAddress1.equals(postalAddress.get("address1")) || lastAddress2 == null || !lastAddress2.equals(postalAddress.get("address2")) || lastCity == null || !lastCity.equals(postalAddress.get("city")) || lastCountryGeoId == null || !lastCountryGeoId.equals(postalAddress.get("countryGeoId"));
                    lastAddress1 = (String) postalAddress.get("address1");
                    lastAddress2 = (String) postalAddress.get("address2");
                    lastCity = (String) postalAddress.get("city");
                    lastCountryGeoId = (String) postalAddress.get("countryGeoId");
                }
                // casting is here necessary for some compiler versions
                Map<String, Object> telecomNumber = UtilMisc.toMap("userLogin", (Object) userLogin);
                Boolean telecomNumberChanged = false;
                if ("TELECOM_NUMBER".equals(currentContactMechTypeId)) {
                    telecomNumber.put("countryCode", rec.get("telCountryCode"));
                    telecomNumber.put("areaCode", rec.get("telAreaCode"));
                    telecomNumber.put("contactNumber", rec.get("telContactNumber"));
                    telecomNumberChanged = lastCountryCode == null || !lastCountryCode.equals(telecomNumber.get("countryCode")) || lastAreaCode == null || !lastAreaCode.equals(telecomNumber.get("areaCode")) || lastContactNumber == null || !lastContactNumber.equals(telecomNumber.get("contactNumber"));
                    lastCountryCode = (String) telecomNumber.get("countryCode");
                    lastAreaCode = (String) telecomNumber.get("areaCode");
                    lastContactNumber = (String) telecomNumber.get("contactNumber");
                }
                Map<String, Object> partyContactMechPurpose = UtilMisc.toMap("partyId", newPartyId, "userLogin", userLogin);
                Boolean partyContactMechPurposeChanged = false;
                currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId");
                if (currentContactMechPurposeTypeId != null && ("TELECOM_NUMBER".equals(currentContactMechTypeId) || "POSTAL_ADDRESS".equals(currentContactMechTypeId) || "EMAIL_ADDRESS".equals(currentContactMechTypeId))) {
                    partyContactMechPurpose.put("contactMechPurposeTypeId", currentContactMechPurposeTypeId);
                    partyContactMechPurposeChanged = (lastContactMechPurposeTypeId == null || !lastContactMechPurposeTypeId.equals(currentContactMechPurposeTypeId)) && !telecomNumberChanged && !postalAddressChanged && !emailAddressChanged;
                    Debug.logInfo("===================================last:" + lastContactMechPurposeTypeId + " current: " + currentContactMechPurposeTypeId + " t :" + telecomNumberChanged + " p: " + postalAddressChanged + " e: " + emailAddressChanged + " result: " + partyContactMechPurposeChanged, module);
                }
                lastContactMechPurposeTypeId = currentContactMechPurposeTypeId;
                // update 
                if (errMsgs.size() == 0) {
                    if (postalAddressChanged) {
                        result = dispatcher.runSync("createPostalAddress", postalAddress);
                        newContactMechId = (String) result.get("contactMechId");
                        if (currentContactMechPurposeTypeId == null) {
                            currentContactMechPurposeTypeId = "GENERAL_LOCATION";
                        }
                        dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
                    }
                    if (telecomNumberChanged) {
                        result = dispatcher.runSync("createTelecomNumber", telecomNumber);
                        newContactMechId = (String) result.get("contactMechId");
                        if (currentContactMechPurposeTypeId == null) {
                            currentContactMechPurposeTypeId = "PHONE_WORK";
                        }
                        dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
                    }
                    if (emailAddressChanged) {
                        result = dispatcher.runSync("createContactMech", emailAddress);
                        newContactMechId = (String) result.get("contactMechId");
                        if (currentContactMechPurposeTypeId == null) {
                            currentContactMechPurposeTypeId = "PRIMARY_EMAIL";
                        }
                        dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
                    }
                    if (partyContactMechPurposeChanged) {
                        partyContactMechPurpose.put("contactMechId", newContactMechId);
                        result = dispatcher.runSync("createPartyContactMechPurpose", partyContactMechPurpose);
                    }
                    lastPartyId = currentPartyId;
                    errMsgs.addAll(newErrMsgs);
                    newErrMsgs = new LinkedList<String>();
                }
            }
        }
    } catch (GenericServiceException e) {
        Debug.logError(e, module);
        return ServiceUtil.returnError(e.getMessage());
    } catch (GenericEntityException e) {
        Debug.logError(e, module);
        return ServiceUtil.returnError(e.getMessage());
    } catch (IOException e) {
        Debug.logError(e, module);
        return ServiceUtil.returnError(e.getMessage());
    }
    if (errMsgs.size() > 0) {
        return ServiceUtil.returnError(errMsgs);
    }
    result = ServiceUtil.returnSuccess(UtilProperties.getMessage(resource, "PartyNewPartiesCreated", UtilMisc.toMap("partiesCreated", partiesCreated), locale));
    return result;
}

private AQuiXEvent load(CSVStreamSource source) throws QuiXException {
    try {
        this.parser = CSVFormat.EXCEL.parse(source.asReader());
        this.iter = this.parser.iterator();
    } catch (IOException e) {
        throw new QuiXException(e);
    }
    this.buffer.add(AQuiXEvent.getStartArray());
    return AQuiXEvent.getStartTable();
}

public static CSVWriter create() {
    return new CSVWriter(CSVFormat.RFC4180);
}

/**
	 * Load a dump file from the given input stream into a queue.
	 *
	 * @param queue the queue to load the dump into
	 * @param input the input stream to load the dump from
	 * @throws IOException if the dump could not be loaded
	 */
private void loadFromCSV(final DocumentFactory factory, final DocumentQueue queue, final InputStream input) throws IOException {
    final String pathField = options.get("pathField").value().orElse("path");
    for (CSVRecord record : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(input))) {
        queue.add(factory.create(Paths.get(record.get(pathField))));
    }
}

public CSVFormat getCSVFormat() {
    return CSVFormat.DEFAULT.withDelimiter(delimiter).withRecordSeparator(recordSeparator).withNullString(nullString).withEscape(escapeCharacter).withQuote(quoteCharacter).withQuoteMode(quoteMode);
}

private CSVParser createCSVParser(final BufferedReader br, final CSVFormat csvFormat) {
    CSVParser parser = null;
    try {
        parser = new CSVParser(br, csvFormat);
    } catch (final IOException e) {
        throw new IllegalArgumentException("Failed to create a CSVParser, because:" + e.getMessage());
    }
    return parser;
}

public static InputStream generateCsvReport(AgreementResult aResult) throws UnsupportedEncodingException, IOException {
    ByteArrayOutputStream buf = new ByteArrayOutputStream();
    try (CSVPrinter printer = new CSVPrinter(new OutputStreamWriter(buf, "UTF-8"), CSVFormat.RFC4180)) {
        AgreementUtils.toCSV(printer, aResult);
    }
    return new ByteArrayInputStream(buf.toByteArray());
}