Java Examples for org.apache.commons.csv.CSVRecord
The following java examples will help you to understand the usage of org.apache.commons.csv.CSVRecord. These source code samples are taken from different open source projects.
Example 1
| Project: DoSeR-master File: WordcountMapper.java View source code |
@Override
public void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes());
int n = value.getLength();
byte[] bytes = new byte[n];
inputStream.read(bytes, 0, n);
CSVParser parser = CSVParser.parse(new String(bytes), CSVFormat.DEFAULT);
try {
for (CSVRecord csvRecord : parser) {
for (String string : csvRecord) {
word.set(string);
context.write(word, one);
Counter counter = context.getCounter(CountersEnum.class.getName(), CountersEnum.INPUT_WORDS.toString());
counter.increment(1);
}
}
} catch (Exception e) {
}
}Example 2
| Project: jat-master File: CSVFileOutputReader.java View source code |
@Override
public List<JATETerm> read(String file) throws IOException {
CSVParser parser = new CSVParser(new FileReader(file), format);
List<JATETerm> out = new ArrayList<>();
for (CSVRecord rec : parser.getRecords()) {
out.add(new JATETerm(rec.get(0).trim(), Double.valueOf(rec.get(1).trim())));
}
Collections.sort(out);
return out;
}Example 3
| Project: mathosphere-master File: PerformanceHelper.java View source code |
private static void runTest(File source, File gold, File expectations, String command) throws Exception {
FileReader in = new FileReader(gold);
Iterable<CSVRecord> records = CSVFormat.RFC4180.withHeader().parse(in);
Set<String> expected = new HashSet<>();
for (CSVRecord record : records) {
expected.add(record.get("identifier"));
}
final byte[] jsonData = Files.readAllBytes(expectations.toPath());
JSONObject jsonObject = (JSONObject) JSONSerializer.toJSON(new String(jsonData));
JSONObject identifiers = jsonObject.getJSONObject("identifiers");
double expPrec = identifiers.getDouble("precision");
double expRec = identifiers.getDouble("recall");
String[] args = { command, "-in", source.getAbsolutePath(), "--tex" };
final PrintStream stdout = System.out;
final ByteArrayOutputStream myOut = new ByteArrayOutputStream();
System.setOut(new PrintStream(myOut));
final long t0 = System.nanoTime();
Main.main(args);
final String standardOutput = myOut.toString();
System.setOut(stdout);
System.out.println((System.nanoTime() - t0) / 1000000000 + "s");
Set<String> sysOut = new HashSet<>(Arrays.asList(standardOutput.split("\r?\n")));
Set<String> real = new HashSet<>();
for (String s : sysOut) {
real.add(s.replaceAll("_\\{(.*?)\\}$", "_"));
}
Set<String> tp = new HashSet<>(expected);
Set<String> fn = new HashSet<>(expected);
Set<String> fp = new HashSet<>(real);
fn.removeAll(real);
fp.removeAll(expected);
tp.retainAll(real);
double rec = ((double) tp.size()) / (tp.size() + fn.size());
double prec = ((double) tp.size()) / (tp.size() + fp.size());
assertThat("precision", prec, Matchers.greaterThan(expPrec));
assertThat("recall", rec, Matchers.greaterThan(expRec));
}Example 4
| Project: data-quality-master File: BigFileAnalyzerPerformanceTest.java View source code |
private static List<String[]> getRecords(String path) {
List<String[]> records = new ArrayList<String[]>();
try {
Reader reader = new FileReader(BigFileAnalyzerPerformanceTest.class.getResource(path).getPath());
CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(';').withFirstRecordAsHeader();
Iterable<CSVRecord> csvRecords = csvFormat.parse(reader);
for (CSVRecord csvRecord : csvRecords) {
String[] values = new String[csvRecord.size()];
for (int i = 0; i < csvRecord.size(); i++) {
values[i] = csvRecord.get(i);
}
records.add(values);
}
} catch (IOException e) {
e.printStackTrace();
}
return records;
}Example 5
| Project: hackday-master File: ExtratorKhan.java View source code |
public static void main(String[] args) throws IOException {
File csvData = new File("khan.csv");
CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8, CSVFormat.RFC4180);
PrintStream out = new PrintStream("khan.sql");
int registro = 1;
for (CSVRecord csvRecord : parser) {
if (registro++ == 1)
continue;
out.print("insert into khan (");
out.print("rede,escola,turma,login,aluno,com_dificuldade,precisa_praticar,praticado,nivel1,nivel2,dominado,pontos,exerciseminutes,videominutes,totalminutes,semana");
out.print(") values (");
out.print("'" + limpaTexto(csvRecord.get(0)) + "',");
out.print("'" + limpaTexto(csvRecord.get(1)) + "',");
out.print("'" + limpaTexto(csvRecord.get(2).replace("'", "")) + "',");
out.print("'" + limpaTexto(csvRecord.get(3)) + "',");
out.print("'" + limpaTexto(csvRecord.get(4)) + "',");
out.print("'" + limpaNumero(csvRecord.get(5)) + "',");
out.print("'" + limpaNumero(csvRecord.get(6)) + "',");
out.print("'" + limpaNumero(csvRecord.get(7)) + "',");
out.print("'" + limpaNumero(csvRecord.get(8)) + "',");
out.print("'" + limpaNumero(csvRecord.get(9)) + "',");
out.print("'" + limpaNumero(csvRecord.get(10)) + "',");
out.print("'" + limpaNumero(csvRecord.get(11)) + "',");
out.print("'" + limpaNumero(csvRecord.get(12)) + "',");
out.print("'" + limpaNumero(csvRecord.get(13)) + "',");
out.print("'" + limpaNumero(csvRecord.get(14)) + "',");
out.print("'" + csvRecord.get(15) + "')");
out.print(";");
out.println();
}
out.close();
}Example 6
| Project: sw360portal-master File: ConvertRecord.java View source code |
public static List<RiskCategory> convertRiskCategories(List<CSVRecord> records) { ArrayList<RiskCategory> list = new ArrayList<>(records.size()); for (CSVRecord record : records) { if (record.size() < 2) break; int id = Integer.parseInt(record.get(0)); String text = record.get(1); RiskCategory category = new RiskCategory().setRiskCategoryId(id).setText(text); list.add(category); } return list; }
Example 7
| Project: Tanaguru-master File: CodeGeneratorMojo.java View source code |
@Override
public void execute() {
try {
isContextValid();
} catch (InvalidParameterException ipe) {
Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ipe);
return;
}
// Initializes engine
VelocityEngine ve = initializeVelocity();
Iterable<CSVRecord> records = getCsv();
if (records == null) {
return;
}
try {
initializeContext();
generate(ve, records);
cleanUpUnusedFiles();
} catch (IOException ex) {
Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
} catch (ResourceNotFoundException ex) {
Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
} catch (ParseErrorException ex) {
Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
} catch (Exception ex) {
Logger.getLogger(CodeGeneratorMojo.class.getName()).log(Level.SEVERE, null, ex);
}
}Example 8
| Project: act-master File: TargetMolecule.java View source code |
public static TargetMolecule fromCSVRecord(CSVRecord record) throws MolFormatException {
String inchi = record.get("inchi");
String displayName = record.get("display_name");
String inchiKey = record.get("inchi_key");
String imageName = record.get("image_name");
Molecule mol = MolImporter.importMol(inchi);
return new TargetMolecule(mol, inchi, displayName, inchiKey, imageName);
}Example 9
| Project: ecs-sync-master File: AbstractStorage.java View source code |
/**
* Default implementation uses a CSV parser to extract the first value, then sets the raw line of text on the summary
* to make it available to other plugins. Note that any overriding implementation *must* catch Exception from
* {@link #createSummary(String)} and return a new zero-sized {@link ObjectSummary} for the identifier
*/
@Override
public ObjectSummary parseListLine(String listLine) {
try {
CSVRecord record = CSVFormat.EXCEL.parse(new StringReader(listLine)).iterator().next();
ObjectSummary summary;
try {
summary = createSummary(record.get(0));
} catch (Exception e) {
summary = new ObjectSummary(record.get(0), false, 0);
}
summary.setListFileRow(listLine);
return summary;
} catch (IOException e) {
throw new RuntimeException("could not parse list-file line", e);
}
}Example 10
| Project: find-master File: PlatformDataExportServiceIT.java View source code |
@Test
public void exportToCsv() throws E, IOException {
final R queryRequest = queryRequestBuilderFactory.getObject().queryRestrictions(testUtils.buildQueryRestrictions()).queryType(QueryRequest.QueryType.MODIFIED).build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
exportService.exportQueryResults(outputStream, queryRequest, ExportFormat.CSV, Collections.emptyList(), 1001L);
final String output = outputStream.toString();
assertNotNull(output);
try (final CSVParser csvParser = CSVParser.parse(output, CSVFormat.EXCEL)) {
final List<CSVRecord> records = csvParser.getRecords();
assertThat(records, not(empty()));
final CSVRecord headerRecord = records.get(0);
// byte-order mark may get in the way
assertThat(headerRecord.get(0), endsWith("Reference"));
assertEquals("Database", headerRecord.get(1));
final CSVRecord firstDataRecord = records.get(1);
final String firstDataRecordReference = firstDataRecord.get(0);
assertNotNull(firstDataRecordReference);
assertFalse(firstDataRecordReference.trim().isEmpty());
final String firstDataRecordDatabase = firstDataRecord.get(1);
assertFalse(firstDataRecordDatabase.trim().isEmpty());
}
}Example 11
| Project: geosummly-master File: EvaluationTools.java View source code |
/**
* Fill in the matrix of aggregate (frequency) values from a list of CSV
* records. The header won't be considered. Timestamp, latitude and
* longitude columns won't be considered.
*/
public ArrayList<ArrayList<Double>> buildAggregatesFromList(List<CSVRecord> list) {
ArrayList<ArrayList<Double>> matrix = new ArrayList<ArrayList<Double>>();
// remove the header, so i=1
for (int k = 1; k < list.size(); k++) {
ArrayList<Double> rec = new ArrayList<Double>();
// remove timestamp, latitude and longitude columns, so j=3
for (int j = 3; j < list.get(k).size(); j++) rec.add(Double.parseDouble(list.get(k).get(j)));
matrix.add(rec);
}
return matrix;
}Example 12
| Project: hadatac-master File: Parser.java View source code |
private ParsingResult indexMeasurements() {
System.out.println("indexMeasurements()...");
String message = "";
try {
files.openFile("csv", "r");
} catch (IOException e) {
e.printStackTrace();
message += "[ERROR] Fail to open the csv file\n";
return new ParsingResult(1, message);
}
Iterable<CSVRecord> records = null;
try {
records = CSVFormat.DEFAULT.withHeader().parse(files.getReader("csv"));
} catch (IOException e) {
e.printStackTrace();
message += "[ERROR] Fail to parse header of the csv file\n";
return new ParsingResult(1, message);
}
int total_count = 0;
int batch_size = 10000;
int nTimeStampCol = -1;
int nTimeInstantCol = -1;
int nIdCol = -1;
for (MeasurementType mt : hadatacKb.getDataset().getMeasurementTypes()) {
if (mt.getTimestampColumn() > -1) {
nTimeStampCol = mt.getTimestampColumn();
}
if (mt.getTimeInstantColumn() > -1) {
nTimeInstantCol = mt.getTimeInstantColumn();
}
if (mt.getIdColumn() > -1) {
nIdCol = mt.getIdColumn();
}
}
boolean isSubjectPlatform = Subject.isPlatform(hadatacKb.getDeployment().getPlatform().getUri());
SolrClient solr = new HttpSolrClient(Play.application().configuration().getString("hadatac.solr.data") + Collections.DATA_ACQUISITION);
for (CSVRecord record : records) {
Iterator<MeasurementType> iter = hadatacKb.getDataset().getMeasurementTypes().iterator();
while (iter.hasNext()) {
MeasurementType measurementType = iter.next();
if (measurementType.getTimestampColumn() > -1) {
continue;
}
if (measurementType.getTimeInstantColumn() > -1) {
continue;
}
if (measurementType.getIdColumn() > -1) {
continue;
}
Measurement measurement = new Measurement();
if (record.get(measurementType.getValueColumn() - 1).isEmpty()) {
continue;
} else {
String originalValue = record.get(measurementType.getValueColumn() - 1);
String codeValue = Subject.findCodeValue(measurementType.getCharacteristicUri(), originalValue);
if (null == codeValue) {
measurement.setValue(originalValue);
} else {
measurement.setValue(codeValue);
}
}
if (nTimeStampCol > -1) {
String sTime = record.get(nTimeStampCol - 1);
int timeStamp = new BigDecimal(sTime).intValue();
Date time = new Date((long) timeStamp * 1000);
measurement.setTimestamp(time.toString());
} else if (nTimeInstantCol > -1) {
measurement.setTimestamp(record.get(nTimeInstantCol - 1));
} else {
measurement.setTimestamp("");
}
measurement.setStudyUri(ValueCellProcessing.replaceNameSpaceEx(hadatacKb.getDataAcquisition().getStudyUri()));
if (nIdCol > -1) {
if (measurementType.getEntityUri().equals(ValueCellProcessing.replacePrefixEx("sio:Human"))) {
Subject subject = Subject.findSubject(measurement.getStudyUri(), record.get(nIdCol - 1));
if (null != subject) {
String subjectUri = subject.getUri();
subjectUri = Subject.checkObjectUri(subjectUri, measurementType.getCharacteristicUri());
measurement.setObjectUri(subjectUri);
} else {
measurement.setObjectUri("");
}
} else if (measurementType.getEntityUri().equals(ValueCellProcessing.replacePrefixEx("sio:Sample"))) {
String sampleUri = Subject.findSampleUri(measurement.getStudyUri(), record.get(nIdCol - 1));
if (sampleUri != null) {
measurement.setObjectUri(sampleUri);
} else {
measurement.setObjectUri("");
}
}
} else {
if (isSubjectPlatform) {
measurement.setObjectUri(hadatacKb.getDeployment().getPlatform().getUri());
} else {
measurement.setObjectUri("");
}
}
measurement.setUri(ValueCellProcessing.replacePrefixEx(measurement.getStudyUri()) + "/" + ValueCellProcessing.replaceNameSpaceEx(hadatacKb.getDataAcquisition().getUri()).split(":")[1] + "/" + hadatacCcsv.getDataset().getLocalName() + "/" + measurementType.getLocalName() + "-" + total_count);
measurement.setOwnerUri(hadatacKb.getDataAcquisition().getOwnerUri());
measurement.setAcquisitionUri(hadatacKb.getDataAcquisition().getUri());
measurement.setUnit(measurementType.getUnitLabel());
measurement.setUnitUri(measurementType.getUnitUri());
measurement.setCharacteristic(measurementType.getCharacteristicLabel());
measurement.setCharacteristicUri(measurementType.getCharacteristicUri());
measurement.setInstrumentModel(hadatacKb.getDeployment().getInstrument().getLabel());
measurement.setInstrumentUri(hadatacKb.getDeployment().getInstrument().getUri());
measurement.setPlatformName(hadatacKb.getDeployment().getPlatform().getLabel());
measurement.setPlatformUri(hadatacKb.getDeployment().getPlatform().getUri());
measurement.setEntity(measurementType.getEntityLabel());
measurement.setEntityUri(measurementType.getEntityUri());
measurement.setDatasetUri(hadatacCcsv.getDatasetKbUri());
try {
solr.addBean(measurement);
} catch (IOExceptionSolrServerException | e) {
System.out.println("[ERROR] SolrClient.addBean - e.Message: " + e.getMessage());
}
if ((++total_count) % batch_size == 0) {
try {
System.out.println("solr.commit()...");
solr.commit();
System.out.println(String.format("Committed %s measurements!", batch_size));
} catch (IOExceptionSolrServerException | e) {
System.out.println("[ERROR] SolrClient.commit - e.Message: " + e.getMessage());
message += "[ERROR] Fail to commit to solr\n";
try {
solr.close();
} catch (IOException e1) {
System.out.println("[ERROR] SolrClient.close - e.Message: " + e1.getMessage());
message += "[ERROR] Fail to close solr\n";
}
return new ParsingResult(1, message);
}
}
}
}
try {
try {
System.out.println("solr.commit()...");
solr.commit();
System.out.println(String.format("Committed %s measurements!", total_count % batch_size));
} catch (IOExceptionSolrServerException | e) {
solr.close();
System.out.println("[ERROR] SolrClient.commit - e.Message: " + e.getMessage());
message += "[ERROR] Fail to commit to solr\n";
return new ParsingResult(1, message);
}
files.closeFile("csv", "r");
} catch (IOException e) {
e.printStackTrace();
message += "[ERROR] Fail to close the csv file\n";
return new ParsingResult(1, message);
}
hadatacKb.getDataAcquisition().addNumberDataPoints(total_count);
hadatacKb.getDataAcquisition().save();
System.out.println("Finished indexMeasurements()");
try {
solr.close();
} catch (IOException e) {
System.out.println("[ERROR] SolrClient.close - e.Message: " + e.getMessage());
message += "[ERROR] Fail to close solr\n";
}
return new ParsingResult(0, message);
}Example 13
| Project: QuiXDM-master File: CSVQuiXEventStreamReader.java View source code |
@Override
protected AQuiXEvent process(CallBack callback) throws QuiXException {
try {
if (!this.buffer.isEmpty()) {
return this.buffer.poll();
}
AQuiXEvent event = null;
if (!this.iter.hasNext() && callback.getState() == State.START_SOURCE) {
// special case if the buffer is empty but the document has not
// been closed
event = AQuiXEvent.getEndArray();
this.buffer.add(AQuiXEvent.getEndTable());
callback.setState(State.END_SOURCE);
return event;
}
if (callback.getState() == State.END_SOURCE) {
return callback.processEndSource();
}
// this iter has next
CSVRecord next = this.iter.next();
for (String cell : next) {
this.buffer.add(AQuiXEvent.getValueString(QuiXCharStream.fromSequence(cell)));
}
this.buffer.add(AQuiXEvent.getEndArray());
return AQuiXEvent.getStartArray();
} catch (Exception e) {
throw new QuiXException(e);
}
}Example 14
| Project: rival-master File: UIPParser.java View source code |
/**
* {@inheritDoc}
*/
@Override
public TemporalDataModelIF<Long, Long> parseTemporalData(final File f) throws IOException {
TemporalDataModelIF<Long, Long> dataset = new TemporalDataModel<>();
Reader in = new InputStreamReader(new FileInputStream(f), "UTF-8");
Iterable<CSVRecord> records;
if (isHasHeader()) {
records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).withHeader().parse(in);
} else {
records = CSVFormat.EXCEL.withDelimiter(getDelimiter()).parse(in);
}
for (CSVRecord record : records) {
long userID = Long.parseLong(record.get(getUserTok()));
long itemID = Long.parseLong(record.get(getItemTok()));
long timestamp = -1L;
if (getTimeTok() != -1) {
timestamp = Long.parseLong(record.get(getTimeTok()));
}
double preference = Double.parseDouble(record.get(getPrefTok()));
dataset.addPreference(userID, itemID, preference);
dataset.addTimestamp(userID, itemID, timestamp);
}
in.close();
return dataset;
}Example 15
| Project: WhiteRabbit-master File: Database.java View source code |
public static Database generateModelFromCSV(InputStream stream, String dbName) {
Database database = new Database();
database.dbName = dbName.substring(0, dbName.lastIndexOf("."));
Map<String, Table> nameToTable = new HashMap<String, Table>();
try {
for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(stream))) {
Table table = nameToTable.get(row.get("TABLE_NAME").toLowerCase());
if (table == null) {
table = new Table();
table.setDb(database);
table.setName(row.get("TABLE_NAME").toLowerCase());
nameToTable.put(row.get("TABLE_NAME").toLowerCase(), table);
database.tables.add(table);
}
Field field = new Field(row.get("COLUMN_NAME").toLowerCase(), table);
field.setNullable(row.get("IS_NULLABLE").equals("YES"));
field.setType(row.get("DATA_TYPE"));
field.setDescription(row.get("DESCRIPTION"));
table.getFields().add(field);
}
} catch (IOException e) {
throw new RuntimeException(e.getMessage());
}
return database;
}Example 16
| Project: ambari-master File: CsvSerializerTest.java View source code |
@Test
public void testSerializeResources_NoColumnInfo() throws Exception {
Result result = new ResultImpl(true);
result.setResultStatus(new ResultStatus(ResultStatus.STATUS.OK));
TreeNode<Resource> tree = result.getResultTree();
List<TreeMap<String, Object>> data = new ArrayList<TreeMap<String, Object>>() {
{
add(new TreeMap<String, Object>() {
{
put("property1", "value1a");
put("property2", "value2a");
put("property3", "value3a");
put("property4", "value4a");
}
});
add(new TreeMap<String, Object>() {
{
put("property1", "value1'b");
put("property2", "value2'b");
put("property3", "value3'b");
put("property4", "value4'b");
}
});
add(new TreeMap<String, Object>() {
{
put("property1", "value1,c");
put("property2", "value2,c");
put("property3", "value3,c");
put("property4", "value4,c");
}
});
}
};
tree.setName("items");
tree.setProperty("isCollection", "true");
addChildResource(tree, "resource", 0, data.get(0));
addChildResource(tree, "resource", 1, data.get(1));
addChildResource(tree, "resource", 2, data.get(2));
replayAll();
//execute test
Object o = new CsvSerializer().serialize(result).toString().replace("\r", "");
verifyAll();
assertNotNull(o);
StringReader reader = new StringReader(o.toString());
CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT);
List<CSVRecord> records = csvParser.getRecords();
assertNotNull(records);
assertEquals(3, records.size());
int i = 0;
for (CSVRecord record : records) {
TreeMap<String, Object> actualData = data.get(i++);
assertEquals(actualData.size(), record.size());
for (String item : record) {
assertTrue(actualData.containsValue(item));
}
}
csvParser.close();
}Example 17
| Project: android-speedtest-mapper-master File: CsvDataParser.java View source code |
/**
* Parses CSV data
*
* @param csvHeader Header items for csv records
* @param csvData
* @return
*/
public static List<SpeedTestRecord> parseCsvData(String csvHeader, String csvData) {
Reader in = new StringReader(getCsvData(csvHeader, csvData));
try {
CSVParser parser = new CSVParser(in, CSVFormat.DEFAULT.toBuilder().withHeader().build());
// get the parsed records
List<CSVRecord> list = parser.getRecords();
// create a list to convert data into SpeedTestRecord model
List<SpeedTestRecord> speedTestRecords = new ArrayList<SpeedTestRecord>();
for (CSVRecord csvRecord : list) {
speedTestRecords.add(new SpeedTestRecord(csvRecord));
}
return speedTestRecords;
} catch (IOException e) {
Tracer.error(LOG_TAG, e);
}
// when no data, send empty list
return Collections.emptyList();
}Example 18
| Project: aw-reporting-master File: AwReportCsvReader.java View source code |
/**
* Returns the next CSV line in the file. If the line starts with the AW last line total String,
* then returns null.
*
* @return the next line in the CSV, or {@code null} in case the file has ended, or the total line
* was reached
*/
public String[] readNext() {
String[] next = null;
if (records.hasNext()) {
CSVRecord record = records.next();
List<String> values = Lists.newArrayListWithCapacity(record.size());
for (int i = 0; i < record.size(); i++) {
values.add(record.get(i));
}
if (!hasSummary || !AW_REPORT_CSV_TOTAL.equalsIgnoreCase(Iterables.getFirst(values, null))) {
next = values.toArray(new String[0]);
}
}
return next;
}Example 19
| Project: Decision-master File: DataFlowFromCsvMain.java View source code |
public static void main(String[] args) throws IOException, NumberFormatException, InterruptedException {
if (args.length < 4) {
log.info("Usage: \n param 1 - path to file \n param 2 - stream name to send the data \n param 3 - time in ms to wait to send each data \n param 4 - broker list");
} else {
Producer<String, String> producer = new Producer<String, String>(createProducerConfig(args[3]));
Gson gson = new Gson();
Reader in = new FileReader(args[0]);
CSVParser parser = CSVFormat.DEFAULT.parse(in);
List<String> columnNames = new ArrayList<>();
for (CSVRecord csvRecord : parser.getRecords()) {
if (columnNames.size() == 0) {
Iterator<String> iterator = csvRecord.iterator();
while (iterator.hasNext()) {
columnNames.add(iterator.next());
}
} else {
StratioStreamingMessage message = new StratioStreamingMessage();
message.setOperation(STREAM_OPERATIONS.MANIPULATION.INSERT.toLowerCase());
message.setStreamName(args[1]);
message.setTimestamp(System.currentTimeMillis());
message.setSession_id(String.valueOf(System.currentTimeMillis()));
message.setRequest_id(String.valueOf(System.currentTimeMillis()));
message.setRequest("dummy request");
List<ColumnNameTypeValue> sensorData = new ArrayList<>();
for (int i = 0; i < columnNames.size(); i++) {
// Workaround
Object value = null;
try {
value = Double.valueOf(csvRecord.get(i));
} catch (NumberFormatException e) {
value = csvRecord.get(i);
}
sensorData.add(new ColumnNameTypeValue(columnNames.get(i), null, value));
}
message.setColumns(sensorData);
String json = gson.toJson(message);
log.info("Sending data: {}", json);
producer.send(new KeyedMessage<String, String>(InternalTopic.TOPIC_DATA.getTopicName(), STREAM_OPERATIONS.MANIPULATION.INSERT, json));
log.info("Sleeping {} ms...", args[2]);
Thread.sleep(Long.valueOf(args[2]));
}
}
log.info("Program completed.");
}
}Example 20
| Project: fullstop-master File: CredentialReportCSVParserImpl.java View source code |
private CSVReportEntry toCSVReportEntry(final CSVRecord record) {
final String user = Optional.ofNullable(record.get("user")).orElse("");
final String arn = Optional.ofNullable(record.get("arn")).orElse("");
final Boolean passwordEnabled = Optional.ofNullable(record.get("password_enabled")).map(Boolean::valueOf).orElse(false);
final Boolean mfaActive = Optional.ofNullable(record.get("mfa_active")).map(Boolean::valueOf).orElse(false);
final Boolean accessKey1Active = Optional.ofNullable(record.get("access_key_1_active")).map(Boolean::valueOf).orElse(false);
final Boolean accessKey2Active = Optional.ofNullable(record.get("access_key_2_active")).map(Boolean::valueOf).orElse(false);
return new CSVReportEntry(user, arn, passwordEnabled, mfaActive, accessKey1Active, accessKey2Active);
}Example 21
| Project: geode-social-demo-master File: Application.java View source code |
public void submitPosts() throws IOException, InterruptedException {
List<String> names = loadNames();
for (String name : names) {
people.save(new Person(name));
}
Random random = new Random();
Reader in = new FileReader(new File("data", "/Sentiment Analysis Dataset.csv"));
// Some of the data has comments. Don't use any comment characters or escape
// characters
CSVParser tweets = CSVFormat.newFormat(',').withHeader().parse(in);
int tweetCount = 0;
for (CSVRecord tweet : tweets) {
String name = names.get(random.nextInt(names.size()));
String post = tweet.get("SentimentText");
posts.save(new Post(name, post));
tweetCount++;
System.out.println(name + ": " + post);
Thread.sleep(1000);
if (System.in.available() > 0) {
System.out.println("Exiting...");
System.out.println("Posted " + tweetCount + " tweets");
break;
}
}
}Example 22
| Project: JAVMovieScraper-master File: WebReleaseRenamer.java View source code |
@Override
public String getCleanName(String filename) {
String cleanFileName = filename.toLowerCase();
cleanFileName = replaceSeperatorsWithSpaces(cleanFileName);
/*
* remove things from the filename which are usually not part of the
* scene / movie name such as par2, xvid, divx, etc
*/
//add a space at the end so we our regex works in the next step for the last word
cleanFileName += " ";
for (CSVRecord wordsToRemove : removeTheseWords) {
//putting spaces in front of it so we only get an actual word, not parts of a word
String wordToRemove = wordsToRemove.get(0).toLowerCase();
cleanFileName = cleanFileName.replaceFirst("\\b" + wordToRemove + "\\b", "");
}
cleanFileName = cleanFileName.trim();
/*
* often times files are released with abbreviations in their name which
* messes up doing google searches on them, so we'll do a substitution to get the full name
*/
boolean doneReplacingabbreviation = false;
for (CSVRecord siteNameReplacement : replaceFirstInstanceOfTheseWords) {
/*
* Our format in this file is that the first word on each line is the full name
* of the abbreviation and each subsequent comma seperated entry on the line
* is an abbreviation
*/
String fullSiteName = siteNameReplacement.get(0);
//WebReleaseRenamer.System.out.println("FullSiteName = " + fullSiteName);
for (String abbreviation : siteNameReplacement) {
abbreviation = abbreviation.replace("\"", "");
//System.out.println("abbreviation = " + abbreviation.trim().toLowerCase());
if (cleanFileName.startsWith(abbreviation.trim().toLowerCase() + " ") && abbreviation.trim().length() > 0) {
//System.out.println("Found match = " + abbreviation.trim().toLowerCase());
cleanFileName = cleanFileName.replaceFirst(Pattern.quote(abbreviation.trim().toLowerCase() + " "), fullSiteName + " ");
doneReplacingabbreviation = true;
//just assume we want to only replace one abbreviaton
break;
}
//System.out.println("CFN: " + cleanFileName);
}
if (doneReplacingabbreviation)
break;
//System.out.println(siteNameReplacement);
}
//Fix up the case and trim it - not needed for search but it just looks better :)
cleanFileName = WordUtils.capitalize(cleanFileName).trim();
return cleanFileName;
}Example 23
| Project: structr-master File: FromCsvFunction.java View source code |
@Override
public Object apply(final ActionContext ctx, final Object caller, final Object[] sources) {
if (arrayHasMinLengthAndMaxLengthAndAllElementsNotNull(sources, 1, 4)) {
try {
final List<Map<String, String>> objects = new LinkedList<>();
final String source = sources[0].toString();
String delimiter = ";";
String quoteChar = "\"";
String recordSeparator = "\n";
switch(sources.length) {
case 4:
recordSeparator = (String) sources[3];
case 3:
quoteChar = (String) sources[2];
case 2:
delimiter = (String) sources[1];
break;
}
CSVFormat format = CSVFormat.newFormat(delimiter.charAt(0)).withHeader();
format = format.withQuote(quoteChar.charAt(0));
format = format.withRecordSeparator(recordSeparator);
format = format.withIgnoreEmptyLines(true);
format = format.withIgnoreSurroundingSpaces(true);
format = format.withSkipHeaderRecord(true);
format = format.withQuoteMode(QuoteMode.ALL);
CSVParser parser = new CSVParser(new StringReader(source), format);
for (final CSVRecord record : parser.getRecords()) {
objects.add(record.toMap());
}
return objects;
} catch (Throwable t) {
logException(t, "{}: Exception for parameter: {}", new Object[] { getName(), getParametersAsString(sources) });
}
return "";
} else {
logParameterError(caller, sources, ctx.isJavaScriptContext());
}
return usage(ctx.isJavaScriptContext());
}Example 24
| Project: datacollector-master File: CsvParser.java View source code |
public String[] read() throws IOException {
if (closed) {
throw new IOException("Parser has been closed");
}
if (iterator == null) {
iterator = parser.iterator();
nextRecord = nextRecord();
}
CSVRecord record = nextRecord;
if (nextRecord != null) {
nextRecord = nextRecord();
}
long prevPos = currentPos;
currentPos = (nextRecord != null) ? nextRecord.getCharacterPosition() + skipLinesPosCorrection : reader.getPos();
if (maxObjectLen > -1) {
if (currentPos - prevPos > maxObjectLen) {
ExceptionUtils.throwUndeclared(new ObjectLengthException(Utils.format("CSV Object at offset '{}' exceeds max length '{}'", prevPos, maxObjectLen), prevPos));
}
}
return toArray(record);
}Example 25
| Project: desktop-crm-master File: LeadImporter.java View source code |
public static Collection<Lead> importFile(File f) {
try {
Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(new FileReader(f));
Iterator<CSVRecord> lines = records.iterator();
if (!lines.hasNext())
return null;
CSVRecord headerRow = lines.next();
List<Lead> leads = new ArrayList<>();
while (lines.hasNext()) {
final Iterator<String> content = lines.next().iterator();
final Iterator<String> header = headerRow.iterator();
Lead lead = new Lead(null, "new lead");
StringBuilder desc = new StringBuilder();
// use capterra as default campaign since it doesn't reference
// itself in the import file
Campaign camp = CrmManager.getCampaignByName("Capterra");
if (camp != null)
lead.setCampaignId(camp.getId());
int column = 0;
while (header.hasNext() && content.hasNext()) {
column++;
String key = header.next();
String value = content.next();
System.out.println(key + "=" + value);
if (column == 1 && key.startsWith("RFQ")) {
camp = CrmManager.getCampaignByName("BuyerZone");
if (camp != null)
lead.setCampaignId(camp.getId());
}
switch(key) {
case "Campaign Code":
camp = CrmManager.getCampaignByName(value);
if (camp != null)
lead.setCampaignId(camp.getId());
break;
case "Organization":
case "Company":
lead.setAccountName(value);
break;
case "Seqment":
desc.append(value).append("\n\n");
break;
case "Size":
desc.append("Size: ").append(value).append("\n");
break;
case "Size 2":
case "# of Employees":
desc.append("Staff: ").append(value).append("\n");
break;
case "Size 3":
desc.append("Terminals: ").append(value).append("\n");
break;
case "Applications":
desc.append("Applications: ").append(value).append("\n");
break;
case "Deployment":
desc.append("Deployment: ").append(value).append("\n");
break;
case "Call Notes":
case "Problem Buyer Needs to Solve":
desc.append("\n\nCALL NOTES\n\n").append(value).append("\n\n");
break;
case "Budget Details":
desc.append("\n\nBudget Details\n\n").append(value).append("\n\n");
case "Buyer's Requirements":
desc.append("\n\nBudget Details\n\n").append(value).append("\n\n");
break;
case "Timeframe":
case "Purchase Timeframe":
desc.append("Timeframe: ").append(value).append("\n");
break;
case "Stage in Buying Process":
desc.append("Stage in Buying Process: ").append(value).append("\n");
break;
case "Product":
case "Type of Software Needed":
desc.append("Product: ").append(value).append("\n");
break;
case "Current Software":
desc.append("Current Software: ").append(value).append("\n");
break;
case "Buyer Has Budget":
desc.append("Buyer Has Budget: ").append(value).append("\n");
break;
case "Budget Amount":
desc.append("Budget Amount: ").append(value).append("\n");
break;
case "Decision Maker":
desc.append("Decision Maker: ").append(value).append("\n");
break;
case "Name":
case "Contact Name":
String[] split = value.split(" ");
if (split.length < 2)
lead.setLastName(value);
else {
lead.setFirstname(split[0]);
int i = 0;
StringBuilder sb = new StringBuilder();
for (String string : split) {
i++;
if (i == 1)
continue;
sb.append(string);
if (i < split.length)
sb.append(' ');
}
lead.setLastName(sb.toString());
}
break;
case "First Name":
lead.setFirstname(value);
break;
case "Last Name":
lead.setLastName(value);
break;
case "Job Title":
case "Title":
lead.setJobTitle(value);
break;
case "Phone":
lead.setPhone(value);
break;
case "Email":
lead.setEmail(value);
break;
case "Address":
lead.setAddress(value);
break;
case "City":
lead.setCity(value);
break;
case "State":
lead.setState(value);
break;
case "ZIP Code":
case "ZIP":
case "Zip":
lead.setZip(value);
break;
case "Country":
case "Location":
lead.setCountry(value);
break;
case "Request":
desc.append("Request: ").append(value).append("\n");
break;
case "Timestamp":
desc.append("Timestamp: ").append(value).append("\n");
break;
case "Appointment":
desc.append("\nAppointment: ").append(value).append("\n\n");
break;
default:
desc.append("").append(key).append(": ").append(value).append("\n\n");
break;
}
}
lead.setDescription(desc.toString());
lead.setType("Customer");
leads.add(CrmManager.saveLead(lead));
}
return leads;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}Example 26
| Project: hibiscus.depotviewer-master File: CSVImportConfigDialog.java View source code |
private void reload() throws RemoteException {
this.getError().setValue("");
boolean enable = true;
list.clear();
header.clear();
Integer headerline = ((Integer) getSkipLines().getValue());
try {
SWTUtil.disposeChildren(this.comp);
comp.setLayoutData(new GridData(GridData.FILL_BOTH));
this.comp.setLayout(new GridLayout());
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName((String) getCharset().getValue())));
String line = null;
int counter = 0;
while ((line = br.readLine()) != null && counter < 30) {
GenericObjectHashMap g = new GenericObjectHashMap();
g.setAttribute("Dateiinhalt", line);
list.add(g);
counter++;
}
header.add("Dateiinhalt");
br.close();
FileInputStream is = new FileInputStream(file);
InputStreamReader isr = new InputStreamReader(is, Charset.forName((String) getCharset().getValue()));
CSVFormat format = CSVFormat.RFC4180.withDelimiter(((String) getTrennzeichen().getValue()).charAt(0)).withSkipLines(headerline - 1);
CSVParser parser = new CSVParser(isr, format);
list.clear();
boolean iserror = false;
boolean isheader = true;
header.clear();
for (CSVRecord record : parser) {
if (isheader) {
for (int i = 0; i < record.size(); i++) {
String name = record.get(i);
if (name.isEmpty()) {
name = "Namenlos";
}
String neuername = name;
counter = 1;
while (header.contains(neuername)) {
neuername = name + " (" + counter + ")";
counter++;
}
header.add(neuername);
}
isheader = false;
continue;
}
GenericObjectHashMap g = new GenericObjectHashMap();
g.setAttribute("_DEPOTVIEWER_IGNORE", "");
g.setAttribute("_DEPOTVIEWER_IDX", "" + (record.getRecordNumber() + headerline));
for (int i = 0; i < header.size(); i++) {
if (i >= record.size()) {
g.setAttribute("_DEPOTVIEWER_IGNORE", "X");
iserror = true;
continue;
}
g.setAttribute(header.get(i), record.get(i));
}
list.add(g);
}
if (iserror) {
getError().setValue("Die mit X markierten Zeilen werden ignoriert.");
}
parser.close();
} catch (Exception e) {
Logger.error("unable to read file", e);
this.getError().setValue("Fehler beim Lesen der Datei:\n" + e.getMessage());
enable = false;
}
TablePart tab = new TablePart(list, null);
tab.addColumn("" + headerline, "_DEPOTVIEWER_IDX");
tab.addColumn("", "_DEPOTVIEWER_IGNORE");
for (String h : header) {
tab.addColumn(h, h);
}
tab.paint(comp);
comp.layout(true);
if (weiterbutton != null) {
weiterbutton.setEnabled(enable);
}
}Example 27
| Project: incubator-samoa-master File: TestUtils.java View source code |
public static void assertResults(File outputFile, org.apache.samoa.TestParams testParams) throws IOException {
LOG.info("Checking results file " + outputFile.getAbsolutePath());
// 1. parse result file with csv parser
Reader in = new FileReader(outputFile);
Iterable<CSVRecord> records = CSVFormat.EXCEL.withSkipHeaderRecord(false).withIgnoreEmptyLines(true).withDelimiter(',').withCommentMarker('#').parse(in);
CSVRecord last = null;
Iterator<CSVRecord> iterator = records.iterator();
CSVRecord header = iterator.next();
// Results of Standard Evaluation have 5 columns, and cv Evaluation have 9 columns
int cvEvaluation = (header.size() == 9) ? 1 : 0;
String cvText = (header.size() == 9) ? "[avg] " : "";
Assert.assertEquals("Unexpected column", org.apache.samoa.TestParams.EVALUATION_INSTANCES, header.get(0).trim());
Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.CLASSIFIED_INSTANCES, header.get(1).trim());
Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.CLASSIFICATIONS_CORRECT, header.get(2 + cvEvaluation).trim());
Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.KAPPA_STAT, header.get(3 + 2 * cvEvaluation).trim());
Assert.assertEquals("Unexpected column", cvText + org.apache.samoa.TestParams.KAPPA_TEMP_STAT, header.get(4 + 3 * cvEvaluation).trim());
// 2. check last line result
while (iterator.hasNext()) {
last = iterator.next();
}
assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getEvaluationInstances(), Float.parseFloat(last.get(0))), testParams.getEvaluationInstances() <= Float.parseFloat(last.get(0)));
assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getClassifiedInstances(), Float.parseFloat(last.get(1))), testParams.getClassifiedInstances() <= Float.parseFloat(last.get(1)));
assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getClassificationsCorrect(), Float.parseFloat(last.get(2 + cvEvaluation))), testParams.getClassificationsCorrect() <= Float.parseFloat(last.get(2 + cvEvaluation)));
assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaStat(), Float.parseFloat(last.get(3 + 2 * cvEvaluation))), testParams.getKappaStat() <= Float.parseFloat(last.get(3 + 2 * cvEvaluation)));
assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaTempStat(), Float.parseFloat(last.get(4 + 3 * cvEvaluation))), testParams.getKappaTempStat() <= Float.parseFloat(last.get(4 + 3 * cvEvaluation)));
}Example 28
| Project: MarsImagesAndroid-master File: Rover.java View source code |
public List<CSVRecord> siteLocationData(int siteIndex) { //return cached results if we have them //TODO invalidate cache when needed (after a day?) List<CSVRecord> cachedLocations = locationsBySite.get(siteIndex); if (cachedLocations != null) return cachedLocations; List<CSVRecord> locations = new ArrayList<>(); URL locationsURL = null; try { String sixDigitSite = String.format("%06d", siteIndex); locationsURL = new URL(getURLPrefix() + "/locations/site_" + sixDigitSite + ".csv"); Log.d(TAG, "location url: %@" + locationsURL); HttpURLConnection urlConnection = (HttpURLConnection) locationsURL.openConnection(); try { InputStream in = new BufferedInputStream(urlConnection.getInputStream()); final Reader reader = new InputStreamReader(in); final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT); try { for (final CSVRecord record : parser) { locations.add(record); } locationsBySite.put(siteIndex, locations); } finally { parser.close(); reader.close(); } } finally { urlConnection.disconnect(); } } catch (MalformedURLException e) { Log.e(TAG, "Badly formatted URL for location manifest: " + locationsURL); } catch (IOException e) { Log.e(TAG, "Error reading from location manifest URL: " + locationsURL); } return locations; }
Example 29
| Project: migration-tools-master File: CsvInput.java View source code |
protected Value[] readRow() {
CSVRecord record = iterator.next();
List<ValueType> valueTypes = getValueTypes();
Value[] values = new Value[valueTypes.size()];
int index = 0;
for (String value : newArrayList(record.iterator())) {
if (doubleQuote.equals(value)) {
value = StringUtils.EMPTY;
} else if (value != null && value.length() == 0) {
value = null;
}
ValueType type = valueTypes.get(index);
type = type != null ? type : STRING;
switch(type) {
case BINARY:
values[index] = binary(BASE64.decode(value));
break;
case STRING:
values[index] = string(value);
break;
}
index++;
}
fill(values, valueTypes, index);
return values;
}Example 30
| Project: nifi-master File: CSVRecordReader.java View source code |
@Override
public Record nextRecord() throws IOException, MalformedRecordException {
final RecordSchema schema = getSchema();
for (final CSVRecord csvRecord : csvParser) {
final Map<String, Object> rowValues = new HashMap<>(schema.getFieldCount());
for (final RecordField recordField : schema.getFields()) {
String rawValue = null;
final String fieldName = recordField.getFieldName();
if (csvRecord.isSet(fieldName)) {
rawValue = csvRecord.get(fieldName);
} else {
for (final String alias : recordField.getAliases()) {
if (csvRecord.isSet(alias)) {
rawValue = csvRecord.get(alias);
break;
}
}
}
if (rawValue == null) {
rowValues.put(fieldName, null);
continue;
}
final Object converted = convert(rawValue, recordField.getDataType(), fieldName);
if (converted != null) {
rowValues.put(fieldName, converted);
}
}
return new MapRecord(schema, rowValues);
}
return null;
}Example 31
| Project: notaql-master File: CSVEngineEvaluator.java View source code |
/**
* Evaluates the given transformation.
*
* This first parses the document (with the first line being the header) and then evaluates on our framework.
*
* TODO: this assumes a header line. It might happen that it is not provided.
*
* @param transformation
* @return
*/
@Override
public JavaRDD<ObjectValue> evaluate(Transformation transformation) {
final SparkTransformationEvaluator evaluator = new SparkTransformationEvaluator(transformation);
final JavaSparkContext sc = NotaQL.SparkFactory.getSparkContext();
final CSVFormat format = CSVFormat.DEFAULT;
final JavaRDD<String> csv = sc.textFile(path);
final String first = csv.first();
final CSVRecord header;
try {
header = format.parse(new StringReader(first)).iterator().next();
} catch (IOException e) {
e.printStackTrace();
throw new AssertionError("Header could not be read for some reason.");
}
String[] headerCols = new String[header.size()];
for (int i = 0; i < header.size(); i++) {
headerCols[i] = header.get(i);
}
final CSVFormat headerFormat = CSVFormat.DEFAULT.withHeader(headerCols);
final JavaRDD<CSVRecord> records = csv.filter( f -> !f.equals(first)).map( line -> headerFormat.parse(new StringReader(line)).iterator().next());
final JavaRDD<Value> converted = records.map(ValueConverter::convertToNotaQL);
final JavaRDD<Value> filtered = converted.filter( o -> transformation.satisfiesInPredicate((ObjectValue) o));
return evaluator.process(filtered);
}Example 32
| Project: nuxeo-master File: UserProfileImporter.java View source code |
public void doImport(CoreSession session, CSVParser parser, UserProfileService userProfileService) throws IOException {
log.info(String.format("Importing CSV file: %s", dataFileName));
DocumentType docType = Framework.getLocalService(SchemaManager.class).getDocumentType(UserProfileConstants.USER_PROFILE_DOCTYPE);
if (docType == null) {
log.error("The type " + UserProfileConstants.USER_PROFILE_DOCTYPE + " does not exist");
return;
}
Map<String, Integer> header = parser.getHeaderMap();
if (header == null) {
log.error("No header line, empty file?");
return;
}
Integer nameIndex = header.get(UserProfileImporter.USER_PROFILE_IMPORTER_USERNAME_COL);
if (nameIndex == null) {
log.error("Missing 'username' column");
return;
}
long docsUpdatedCount = 0;
totalRecords = parser.getRecordNumber();
try {
int batchSize = config.getBatchSize();
long lineNumber = 0;
for (CSVRecord record : parser.getRecords()) {
lineNumber++;
currentRecord = lineNumber;
try {
if (importLine(record, lineNumber, nameIndex, docType, session, userProfileService, header)) {
docsUpdatedCount++;
if (docsUpdatedCount % batchSize == 0) {
commitOrRollbackTransaction();
startTransaction();
}
}
} catch (NuxeoException e) {
Throwable unwrappedException = unwrapException(e);
logImportError(lineNumber, "Error while importing line: %s", unwrappedException.getMessage());
log.debug(unwrappedException, unwrappedException);
}
}
session.save();
} finally {
commitOrRollbackTransaction();
startTransaction();
}
log.info(String.format("Done importing %s entries from CSV file: %s", docsUpdatedCount, dataFileName));
}Example 33
| Project: performance-plugin-master File: JMeterCsvParser.java View source code |
protected void parseCSV(Reader in, String[] header, PerformanceReport report) throws IOException {
CSVFormat csvFormat = CSVFormat.newFormat(delimiter).withHeader(header).withQuote('"').withSkipHeaderRecord();
Iterable<CSVRecord> records = csvFormat.parse(in);
for (CSVRecord record : records) {
final HttpSample sample = getSample(record);
try {
report.addSample(sample);
} catch (SAXException e) {
throw new RuntimeException("Error parsing file '" + report.getReportFileName() + "': Unable to add sample for CSVRecord " + record, e);
}
}
}Example 34
| Project: pinot-master File: DataFrame.java View source code |
/* **************************************************************************
* DataFrame parsers
***************************************************************************/
/**
* Reads in a CSV structured stream and returns it as a DataFrame. The native series type is
* chosen to be as specific as possible based on the data ingested.
* <br/><b>NOTE:</b> Expects the first line to contain
* column headers. The column headers are transformed into series names by replacing non-word
* character sequences with underscores ({@code "_"}). Leading digits in series names are also
* escaped with a leading underscore.
*
* @param in input reader
* @return CSV as DataFrame
* @throws IOException if a read error is encountered
* @throws IllegalArgumentException if the column headers cannot be transformed into valid series names
*/
public static DataFrame fromCsv(Reader in) throws IOException {
Iterator<CSVRecord> it = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in).iterator();
if (!it.hasNext())
return new DataFrame();
CSVRecord first = it.next();
Set<String> headers = first.toMap().keySet();
// transform column headers into series names
Map<String, String> header2name = new HashMap<>();
for (String h : headers) {
// remove spaces
String name = Pattern.compile("\\W+").matcher(h).replaceAll("_");
// underscore escape leading number
if (Pattern.compile("\\A[0-9]").matcher(name).find())
name = "_" + name;
if (!SERIES_NAME_PATTERN.matcher(name).matches()) {
throw new IllegalArgumentException(String.format("Series name must match pattern '%s'", SERIES_NAME_PATTERN));
}
header2name.put(h, name);
}
// read first line and initialize builders
Map<String, StringSeries.Builder> builders = new HashMap<>();
for (String h : headers) {
StringSeries.Builder builder = StringSeries.builder();
builder.addValues(first.get(h));
builders.put(h, builder);
}
while (it.hasNext()) {
CSVRecord record = it.next();
for (String h : headers) {
String value = record.get(h);
builders.get(h).addValues(value);
}
}
// construct dataframe and detect native data types
DataFrame df = new DataFrame();
for (Map.Entry<String, StringSeries.Builder> e : builders.entrySet()) {
StringSeries s = e.getValue().build();
Series conv = s.get(s.inferType());
String name = header2name.get(e.getKey());
df.addSeries(name, conv);
}
return df;
}Example 35
| Project: samoa-master File: TestUtils.java View source code |
public static void assertResults(File outputFile, com.yahoo.labs.samoa.TestParams testParams) throws IOException {
LOG.info("Checking results file " + outputFile.getAbsolutePath());
// 1. parse result file with csv parser
Reader in = new FileReader(outputFile);
Iterable<CSVRecord> records = CSVFormat.EXCEL.withSkipHeaderRecord(false).withIgnoreEmptyLines(true).withDelimiter(',').withCommentMarker('#').parse(in);
CSVRecord last = null;
Iterator<CSVRecord> iterator = records.iterator();
CSVRecord header = iterator.next();
Assert.assertEquals("Invalid number of columns", 5, header.size());
Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.EVALUATION_INSTANCES, header.get(0).trim());
Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.CLASSIFIED_INSTANCES, header.get(1).trim());
Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.CLASSIFICATIONS_CORRECT, header.get(2).trim());
Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.KAPPA_STAT, header.get(3).trim());
Assert.assertEquals("Unexpected column", com.yahoo.labs.samoa.TestParams.KAPPA_TEMP_STAT, header.get(4).trim());
// 2. check last line result
while (iterator.hasNext()) {
last = iterator.next();
}
assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getEvaluationInstances(), Float.parseFloat(last.get(0))), testParams.getEvaluationInstances() <= Float.parseFloat(last.get(0)));
assertTrue(String.format("Unmet threshold expected %d got %f", testParams.getClassifiedInstances(), Float.parseFloat(last.get(1))), testParams.getClassifiedInstances() <= Float.parseFloat(last.get(1)));
assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getClassificationsCorrect(), Float.parseFloat(last.get(2))), testParams.getClassificationsCorrect() <= Float.parseFloat(last.get(2)));
assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaStat(), Float.parseFloat(last.get(3))), testParams.getKappaStat() <= Float.parseFloat(last.get(3)));
assertTrue(String.format("Unmet threshold expected %f got %f", testParams.getKappaTempStat(), Float.parseFloat(last.get(4))), testParams.getKappaTempStat() <= Float.parseFloat(last.get(4)));
}Example 36
| Project: Simple-Workflow-Engine-master File: CSVInputAdapter.java View source code |
/* (non-Javadoc)
* @see com.brightcove.opensource.workflowengine.Actor#run()
*/
public void run() {
Property<String> inputFileProp = getFirstProperty("input-file");
if ((inputFileProp == null) || (inputFileProp.getValue() == null)) {
die("CSVAdapter requires an input-file property.");
}
File inputFile = new File(inputFileProp.getValue());
if (!inputFile.exists()) {
die("[ERR] CSVAdapter input file (" + inputFile.getAbsolutePath() + ") does not exist.");
}
Character delimiter = ',';
Character quote = '"';
String delimiterProp = getFirstPropertyValue("delimiter");
if (delimiterProp != null) {
delimiter = delimiterProp.charAt(0);
}
String quoteProp = getFirstPropertyValue("quote");
if (quoteProp != null) {
quote = quoteProp.charAt(0);
}
CSVRecord headerRow = null;
String headerRowProp = getFirstPropertyValue("header-row");
if (headerRowProp != null) {
Reader reader = new StringReader(headerRowProp);
;
Iterable<CSVRecord> parser = null;
try {
parser = CSVFormat.newBuilder().withDelimiter(delimiter).withQuoteChar(quote).parse(reader);
} catch (Exception e) {
die("[ERR] CSVAdapter can't parse header row (" + headerRowProp + "). Exception caught: '" + e + "'.");
}
for (CSVRecord csvRecord : parser) {
if (headerRow == null) {
headerRow = csvRecord;
}
}
}
Boolean hasHeaderRow = false;
String hasHeaderRowProp = getFirstPropertyValue("has-header-row");
if ("true".equalsIgnoreCase(hasHeaderRowProp)) {
hasHeaderRow = true;
}
if (!hasStarted()) {
FileReader reader = null;
Iterable<CSVRecord> parser = null;
try {
reader = new FileReader(inputFile);
parser = CSVFormat.newBuilder().withDelimiter(delimiter).withQuoteChar(quote).parse(reader);
} catch (Exception e) {
die("[ERR] CSVAdapter can't parse input file (" + inputFile.getAbsolutePath() + "). Exception caught: '" + e + "'.");
}
Integer rowIdx = 0;
for (CSVRecord csvRecord : parser) {
rowIdx++;
if (hasHeaderRow && (rowIdx == 1)) {
if (headerRow == null) {
// Only use header row from file if it wasn't supplied by property
headerRow = csvRecord;
}
} else {
Record record = new Record();
for (Integer idx = 0; idx < csvRecord.size(); idx++) {
String propName = "Column-" + idx;
if ((headerRow != null) && (idx < headerRow.size())) {
propName = headerRow.get(idx);
}
Property<?> prop = new Property<String>(propName, csvRecord.get(idx));
record.addProperty(prop);
}
handleRecord(record);
}
}
}
}Example 37
| Project: autopsy-master File: CreditCards.java View source code |
/**
* Load the BIN range information from disk. If the map has already been
* initialized, don't load again.
*/
private static synchronized void loadBINRanges() {
if (binsLoaded == false) {
try {
//NON-NLS
InputStreamReader in = new InputStreamReader(CreditCards.class.getResourceAsStream("ranges.csv"));
CSVParser rangesParser = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in);
//parse each row and add to range map
for (CSVRecord record : rangesParser) {
/**
* Because ranges.csv allows both 6 and (the newer) 8 digit
* BINs, but we need a consistent length for the range map,
* we pad all the numbers out to 8 digits
*/
//pad start with 0's //NON-NLS
String start = StringUtils.rightPad(record.get("iin_start"), 8, "0");
//if there is no end listed, use start, since ranges will be closed.
//NON-NLS
String end = StringUtils.defaultIfBlank(record.get("iin_end"), start);
//pad end with 9's //NON-NLS
end = StringUtils.rightPad(end, 8, "99");
//NON-NLS
final String numberLength = record.get("number_length");
try {
BINRange binRange = new BINRange(Integer.parseInt(start), Integer.parseInt(end), StringUtils.isBlank(numberLength) ? null : Integer.valueOf(numberLength), //NON-NLS
record.get("scheme"), //NON-NLS
record.get("brand"), //NON-NLS
record.get("type"), //NON-NLS
record.get("country"), //NON-NLS
record.get("bank_name"), //NON-NLS
record.get("bank_url"), //NON-NLS
record.get("bank_phone"), //NON-NLS
record.get("bank_city"));
binRanges.put(Range.closed(binRange.getBINstart(), binRange.getBINend()), binRange);
} catch (NumberFormatException numberFormatException) {
LOGGER.log(Level.WARNING, "Failed to parse BIN range: " + record.toString(), numberFormatException);
}
binsLoaded = true;
}
} catch (IOException ex) {
LOGGER.log(Level.WARNING, "Failed to load BIN ranges form ranges.csv", ex);
MessageNotifyUtil.Notify.warn("Credit Card Number Discovery", "There was an error loading Bank Identification Number information. Accounts will not have their BINs identified.");
}
}
}Example 38
| Project: aws-big-data-blog-master File: LambdaContainer.java View source code |
//Validation/Conversion Layer function
public void validateAndNormalizeInputData(S3Event event, Context ctx) throws Exception {
AmazonS3 s3Client;
InputStream inputFileStream = null;
InputStream readableDataStream = null;
List<S3EventNotificationRecord> notificationRecords = event.getRecords();
s3Client = new AmazonS3Client();
String eventFileName, siteName, dbfName;
CSVParser fileParser = null;
for (S3EventNotificationRecord record : notificationRecords) {
eventFileName = record.getS3().getObject().getKey();
S3Object s3Object = s3Client.getObject(new GetObjectRequest(record.getS3().getBucket().getName(), record.getS3().getObject().getKey()));
inputFileStream = s3Object.getObjectContent();
fileParser = new CSVParser(new InputStreamReader(inputFileStream), CSVFormat.TDF.withCommentMarker('-'));
List<CSVRecord> records = fileParser.getRecords();
StringWriter writer = new StringWriter();
CSVPrinter printer = null;
if (records.get(0).toString().matches(".*[^0-9].*")) {
records.remove(0);
}
printer = new CSVPrinter(writer, CSVFormat.DEFAULT.withRecordSeparator(System.getProperty("line.separator")));
printer.printRecords(records);
printer.flush();
readableDataStream = new ByteArrayInputStream(writer.toString().getBytes("utf-8"));
s3Client.putObject(record.getS3().getBucket().getName(), "validated/" + eventFileName + ".csv", readableDataStream, new ObjectMetadata());
printer.close();
readableDataStream.close();
}
}Example 39
| Project: kylo-master File: CSVFileSchemaParser.java View source code |
private DefaultFileSchema populateSchema(CSVParser parser) {
DefaultFileSchema fileSchema = new DefaultFileSchema();
int i = 0;
ArrayList<Field> fields = new ArrayList<>();
for (CSVRecord record : parser) {
if (i > 9) {
break;
}
int size = record.size();
for (int j = 0; j < size; j++) {
DefaultField field = null;
if (i == 0) {
field = new DefaultField();
if (headerRow) {
field.setName(record.get(j));
} else {
field.setName("Col_" + (j + 1));
}
fields.add(field);
} else {
try {
field = (DefaultField) fields.get(j);
field.getSampleValues().add(StringUtils.defaultString(record.get(j), ""));
} catch (IndexOutOfBoundsException e) {
LOG.warn("Sample file has potential sparse column problem at row [?] field [?]", i + 1, j + 1);
}
}
}
i++;
}
fileSchema.setFields(fields);
return fileSchema;
}Example 40
| Project: LanguageMemoryApp-master File: Csv2Properties.java View source code |
public void parseInputCSV() throws IOException {
final Reader reader = new InputStreamReader(inputFile.toURL().openStream(), "UTF-8");
Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(reader);
for (CSVRecord record : records) {
String key_name = record.get(KEY_COLUMN);
String en_value = record.get(EN_COLUMN);
String nl_value = record.get(NL_COLUMN);
String de_value = record.get(DE_COLUMN);
System.out.println(key_name);
System.out.println(en_value);
System.out.println(nl_value);
System.out.println(de_value);
translationsEN.put(key_name, en_value);
translationsNL.put(key_name, nl_value);
translationsDE.put(key_name, de_value);
}
}Example 41
| Project: lumify-master File: CsvGraphPropertyWorker.java View source code |
public void processCsvStream(InputStream rawIn, Mapping mapping, GraphPropertyWorkData data) throws IOException {
Reader reader = new InputStreamReader(rawIn);
Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(reader);
State state = new State(mapping, data);
for (CSVRecord record : records) {
if (record.getRecordNumber() <= mapping.getLinesToSkip()) {
continue;
}
state.setRecord(record);
processCsvLine(state);
}
getGraph().flush();
}Example 42
| Project: myria-master File: FileScan.java View source code |
@Override
protected TupleBatch fetchNextReady() throws DbException, IOException {
/* Let's assume that the scanner always starts at the beginning of a line. */
long lineNumberBegin = lineNumber;
while ((buffer.numTuples() < buffer.getBatchSize())) {
lineNumber++;
if (parser.isClosed()) {
break;
}
try {
if (!iterator.hasNext()) {
parser.close();
break;
}
} catch (final RuntimeException e) {
throw new DbException("Error parsing row " + lineNumber, e);
}
CSVRecord record = iterator.next();
if (record.size() != schema.numColumns()) {
throw new DbException("Error parsing row " + lineNumber + ": Found " + record.size() + " column(s) but expected " + schema.numColumns() + " column(s).");
}
for (int column = 0; column < schema.numColumns(); ++column) {
String cell = record.get(column);
try {
switch(schema.getColumnType(column)) {
case BOOLEAN_TYPE:
Float f = Floats.tryParse(cell);
if (f != null) {
buffer.putBoolean(column, f != 0);
} else {
buffer.putBoolean(column, BooleanUtils.toBoolean(cell));
}
break;
case DOUBLE_TYPE:
buffer.putDouble(column, Double.parseDouble(cell));
break;
case FLOAT_TYPE:
buffer.putFloat(column, Float.parseFloat(cell));
break;
case INT_TYPE:
buffer.putInt(column, Integer.parseInt(cell));
break;
case LONG_TYPE:
buffer.putLong(column, Long.parseLong(cell));
break;
case STRING_TYPE:
buffer.putString(column, cell);
break;
case DATETIME_TYPE:
buffer.putDateTime(column, DateTimeUtils.parse(cell));
break;
case BLOB_TYPE:
// read filename
buffer.putBlob(column, getFile(cell));
break;
}
} catch (final IllegalArgumentException e) {
throw new DbException("Error parsing column " + column + " of row " + lineNumber + ", expected type: " + schema.getColumnType(column) + ", scanned value: " + cell, e);
}
}
}
LOGGER.debug("Scanned {} input lines", lineNumber - lineNumberBegin);
return buffer.popAny();
}Example 43
| Project: nuxeo-services-master File: SQLHelper.java View source code |
private void loadData() throws DirectoryException {
log.debug("loading data file: " + dataFileName);
CSVParser csvParser = null;
PreparedStatement ps = null;
try {
InputStream is = getClass().getClassLoader().getResourceAsStream(dataFileName);
if (is == null) {
is = Framework.getResourceLoader().getResourceAsStream(dataFileName);
if (is == null) {
throw new DirectoryException("data file not found: " + dataFileName);
}
}
csvParser = new CSVParser(new InputStreamReader(is, SQL_SCRIPT_CHARSET), CSVFormat.DEFAULT.withDelimiter(characterSeparator).withHeader());
Map<String, Integer> header = csvParser.getHeaderMap();
List<Column> columns = new ArrayList<>();
Insert insert = new Insert(table);
for (String columnName : header.keySet()) {
String trimmedColumnName = columnName.trim();
Column column = table.getColumn(trimmedColumnName);
if (column == null) {
throw new DirectoryException("column not found: " + trimmedColumnName);
}
columns.add(table.getColumn(trimmedColumnName));
insert.addColumn(column);
}
String insertSql = insert.getStatement();
log.debug("insert statement: " + insertSql);
ps = connection.prepareStatement(insertSql);
for (CSVRecord record : csvParser) {
if (record.size() == 0 || record.size() == 1 && StringUtils.isBlank(record.get(0))) {
// empty lines
continue;
}
if (!record.isConsistent()) {
log.error("invalid column count while reading CSV file: " + dataFileName + ", values: " + record);
continue;
}
if (logger.isLogEnabled()) {
List<Serializable> values = new ArrayList<>(header.size());
for (String value : record) {
if (SQL_NULL_MARKER.equals(value)) {
value = null;
}
values.add(value);
}
logger.logSQL(insertSql, values);
}
for (int i = 0; i < header.size(); i++) {
Column column = columns.get(i);
String value = record.get(i);
Serializable v;
try {
if (SQL_NULL_MARKER.equals(value)) {
v = null;
} else if (column.getType().spec == ColumnSpec.STRING) {
v = value;
} else if (column.getType().spec == ColumnSpec.BOOLEAN) {
v = Boolean.valueOf(value);
} else if (column.getType().spec == ColumnSpec.LONG) {
v = Long.valueOf(value);
} else if (column.getType().spec == ColumnSpec.TIMESTAMP) {
Calendar cal = new GregorianCalendar();
cal.setTime(Timestamp.valueOf(value));
v = cal;
} else if (column.getType().spec == ColumnSpec.DOUBLE) {
v = Double.valueOf(value);
} else {
throw new DirectoryException("unrecognized column type: " + column.getType() + ", values: " + record);
}
column.setToPreparedStatement(ps, i + 1, v);
} catch (IllegalArgumentException e) {
throw new DirectoryException(String.format("failed to set column '%s' on table '%s', values: %s", column.getPhysicalName(), table.getPhysicalName(), record), e);
} catch (SQLException e) {
throw new DirectoryException(String.format("Table '%s' initialization failed: %s, values: %s", table.getPhysicalName(), e.getMessage(), record), e);
}
}
ps.execute();
}
} catch (IOException e) {
throw new DirectoryException("Read error while reading data file: " + dataFileName, e);
} catch (SQLException e) {
throw new DirectoryException(String.format("Table '%s' initialization failed: %s", table.getPhysicalName(), e.getMessage()), e);
} finally {
DirectoryException e = new DirectoryException();
try {
if (csvParser != null) {
csvParser.close();
}
} catch (IOException ioe) {
e.addSuppressed(ioe);
}
try {
if (ps != null) {
ps.close();
}
} catch (SQLException sqle) {
e.addSuppressed(sqle);
}
if (e.getSuppressed().length > 0) {
throw e;
}
}
}Example 44
| Project: pippo-master File: CsvEngine.java View source code |
@Override
@SuppressWarnings("unchecked")
public <T> T fromString(String content, Class<T> classOfT) {
if (!classOfT.isArray()) {
if (Collection.class.isAssignableFrom(classOfT)) {
// Collections are NOT supported for deserialization from CSV
throw new RuntimeException("Collection types are not supported. Please specify an array[] type.");
}
throw new RuntimeException(String.format("Array[] types are required. Please specify %s[]", classOfT.getName()));
}
Class<?> objectType = classOfT.getComponentType();
int currentLine = 0;
try (CSVParser parser = new CSVParser(new StringReader(content), getCSVFormat().withHeader())) {
Set<String> columns = parser.getHeaderMap().keySet();
Map<String, Field> fieldMap = getFieldMap(objectType);
Constructor<?> objectConstructor;
try {
objectConstructor = objectType.getConstructor();
} catch (NoSuchMethodException e) {
throw new RuntimeException("A default constructor is required for " + objectType.getName());
}
List objects = new ArrayList<>();
for (CSVRecord record : parser) {
currentLine++;
Object o = objectConstructor.newInstance();
for (String column : columns) {
Field field = fieldMap.get(caseSensitiveFieldNames ? column : column.toLowerCase());
String value = record.get(column);
Object object = objectFromString(value, field.getType());
field.set(o, object);
}
objects.add(o);
}
Object array = Array.newInstance(objectType, objects.size());
for (int i = 0; i < objects.size(); i++) {
Array.set(array, i, objects.get(i));
}
return (T) array;
} catch (Exception e) {
throw new RuntimeException("Failed to parse CSV near line #" + currentLine, e);
}
}Example 45
| Project: qalingo-i18n-master File: LoaderTranslationUtil.java View source code |
public static final void buildMessagesProperties(String currentPath, String project, String filePath, List<String> activedLanguages, String defaultLanguage, String inputEncoding, String outputEncoding) {
try {
String newPath = currentPath + project + LoaderTranslation.PROPERTIES_PATH;
File folderProject = new File(newPath);
if (!folderProject.exists()) {
folderProject.mkdirs();
}
InputStreamReader reader = new InputStreamReader(new FileInputStream(new File(filePath)), inputEncoding);
LOG.info("File CSV encoding: " + inputEncoding);
LOG.info("File properties encoding: " + outputEncoding);
CSVFormat csvFormat = CSVFormat.DEFAULT;
CSVParser readerCSV = new CSVParser(reader, csvFormat);
List<CSVRecord> records = null;
try {
records = readerCSV.getRecords();
} catch (Exception e) {
LOG.error("Failed to load: " + filePath, e);
}
String prefixFileName = "";
CSVRecord firstLineRecord = records.get(0);
String[] prefixFileNameTemp = firstLineRecord.get(0).split("## Filename :");
if (prefixFileNameTemp.length > 1) {
prefixFileName = prefixFileNameTemp[1].trim();
}
String prefixKey = "";
CSVRecord secondLineRecord = records.get(1);
String[] prefixKeyTemp = secondLineRecord.get(0).split("## PrefixKey :");
if (prefixKeyTemp.length > 1) {
prefixKey = prefixKeyTemp[1].trim();
}
Map<String, Integer> availableLanguages = new HashMap<String, Integer>();
for (CSVRecord record : records) {
String firstCell = record.get(0);
if (firstCell.contains("Prefix") && record.size() > 1) {
String secondCell = record.get(1);
if (secondCell.contains("Key")) {
for (int i = 2; i < record.size(); i++) {
String languageCode = record.get(i);
availableLanguages.put(languageCode, new Integer(i));
}
break;
}
}
}
// BUILD DEFAULT FILE
String fileFullPath = newPath + "/" + prefixFileName + ".properties";
Integer defaultLanguagePosition = availableLanguages.get(defaultLanguage);
buildMessagesProperties(records, fileFullPath, prefixKey, defaultLanguage, defaultLanguagePosition, outputEncoding, true);
for (Iterator<String> iterator = availableLanguages.keySet().iterator(); iterator.hasNext(); ) {
String languageCode = (String) iterator.next();
if (activedLanguages.contains(languageCode)) {
Integer languagePosition = availableLanguages.get(languageCode);
String languegFileFullPath = newPath + "/" + prefixFileName + "_" + languageCode + ".properties";
buildMessagesProperties(records, languegFileFullPath, prefixKey, languageCode, languagePosition.intValue(), outputEncoding, false);
}
}
LOG.info(newPath + "/" + prefixFileName + ".properties");
} catch (Exception e) {
LOG.info("Exception", e);
}
}Example 46
| Project: ranger-master File: FileSourceUserGroupBuilder.java View source code |
public Map<String, List<String>> readTextFile(File textFile) throws Exception {
Map<String, List<String>> ret = new HashMap<String, List<String>>();
String delimiter = config.getUserSyncFileSourceDelimiter();
CSVFormat csvFormat = CSVFormat.newFormat(delimiter.charAt(0));
CSVParser csvParser = new CSVParser(new BufferedReader(new FileReader(textFile)), csvFormat);
List<CSVRecord> csvRecordList = csvParser.getRecords();
if (csvRecordList != null) {
for (CSVRecord csvRecord : csvRecordList) {
List<String> groups = new ArrayList<String>();
String user = csvRecord.get(0);
user = user.replaceAll("^\"|\"$", "");
int i = csvRecord.size();
for (int j = 1; j < i; j++) {
String group = csvRecord.get(j);
if (group != null && !group.isEmpty()) {
group = group.replaceAll("^\"|\"$", "");
groups.add(group);
}
}
ret.put(user, groups);
}
}
csvParser.close();
return ret;
}Example 47
| Project: swagger-inflector-master File: SwaggerOperationController.java View source code |
@Override
public Response apply(ContainerRequestContext ctx) {
List<Parameter> parameters = operation.getParameters();
final RequestContext requestContext = createContext(ctx);
String path = ctx.getUriInfo().getPath();
Map<String, Map<String, String>> formMap = new HashMap<String, Map<String, String>>();
Map<String, File> inputStreams = new HashMap<String, File>();
Object[] args = new Object[parameters.size() + 1];
if (parameters != null) {
int i = 0;
args[i] = requestContext;
i += 1;
List<ValidationMessage> missingParams = new ArrayList<ValidationMessage>();
UriInfo uri = ctx.getUriInfo();
String formDataString = null;
String[] parts = null;
Set<String> existingKeys = new HashSet<String>();
for (Iterator<String> x = uri.getQueryParameters().keySet().iterator(); x.hasNext(); ) {
existingKeys.add(x.next() + ": qp");
}
for (Iterator<String> x = uri.getPathParameters().keySet().iterator(); x.hasNext(); ) {
existingKeys.add(x.next() + ": pp");
}
for (Iterator<String> x = ctx.getHeaders().keySet().iterator(); x.hasNext(); ) {
String key = x.next();
// if(!commonHeaders.contains(key))
// existingKeys.add(key);
}
MediaType mt = requestContext.getMediaType();
for (Parameter p : parameters) {
Map<String, String> headers = new HashMap<String, String>();
String name = null;
if (p instanceof FormParameter) {
if (formDataString == null) {
// can only read stream once
if (mt.isCompatible(MediaType.MULTIPART_FORM_DATA_TYPE)) {
// get the boundary
String boundary = mt.getParameters().get("boundary");
if (boundary != null) {
try {
InputStream output = ctx.getEntityStream();
MultipartStream multipartStream = new MultipartStream(output, boundary.getBytes());
boolean nextPart = multipartStream.skipPreamble();
while (nextPart) {
String header = multipartStream.readHeaders();
// process headers
if (header != null) {
CSVFormat format = CSVFormat.DEFAULT.withDelimiter(';').withRecordSeparator("=");
Iterable<CSVRecord> records = format.parse(new StringReader(header));
for (CSVRecord r : records) {
for (int j = 0; j < r.size(); j++) {
String string = r.get(j);
Iterable<CSVRecord> outerString = CSVFormat.DEFAULT.withDelimiter('=').parse(new StringReader(string));
for (CSVRecord outerKvPair : outerString) {
if (outerKvPair.size() == 2) {
String key = outerKvPair.get(0).trim();
String value = outerKvPair.get(1).trim();
if ("name".equals(key)) {
name = value;
}
headers.put(key, value);
} else {
Iterable<CSVRecord> innerString = CSVFormat.DEFAULT.withDelimiter(':').parse(new StringReader(string));
for (CSVRecord innerKVPair : innerString) {
if (innerKVPair.size() == 2) {
String key = innerKVPair.get(0).trim();
String value = innerKVPair.get(1).trim();
if ("name".equals(key)) {
name = value;
}
headers.put(key, value);
}
}
}
}
if (name != null) {
formMap.put(name, headers);
}
}
}
}
String filename = extractFilenameFromHeaders(headers);
if (filename != null) {
try {
File file = new File(Files.createTempDir(), filename);
file.deleteOnExit();
file.getParentFile().deleteOnExit();
FileOutputStream fo = new FileOutputStream(file);
multipartStream.readBodyData(fo);
inputStreams.put(name, file);
} catch (Exception e) {
LOGGER.error("Failed to extract uploaded file", e);
}
} else {
ByteArrayOutputStream bo = new ByteArrayOutputStream();
multipartStream.readBodyData(bo);
String value = bo.toString();
headers.put(name, value);
}
if (name != null) {
formMap.put(name, headers);
}
headers = new HashMap<>();
name = null;
nextPart = multipartStream.readBoundary();
}
} catch (IOException e) {
e.printStackTrace();
}
}
} else {
try {
formDataString = IOUtils.toString(ctx.getEntityStream(), "UTF-8");
parts = formDataString.split("&");
for (String part : parts) {
String[] kv = part.split("=");
existingKeys.add(kv[0] + ": fp");
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
for (Parameter parameter : parameters) {
String in = parameter.getIn();
Object o = null;
try {
if ("formData".equals(in)) {
SerializableParameter sp = (SerializableParameter) parameter;
String name = parameter.getName();
if (mt.isCompatible(MediaType.MULTIPART_FORM_DATA_TYPE)) {
// look in the form map
Map<String, String> headers = formMap.get(name);
if (headers != null && headers.size() > 0) {
if ("file".equals(sp.getType())) {
o = inputStreams.get(name);
} else {
Object obj = headers.get(parameter.getName());
if (obj != null) {
JavaType jt = parameterClasses[i];
Class<?> cls = jt.getRawClass();
List<String> os = Arrays.asList(obj.toString());
try {
o = validator.convertAndValidate(os, parameter, cls, definitions);
} catch (ConversionException e) {
missingParams.add(e.getError());
} catch (ValidationException e) {
missingParams.add(e.getValidationMessage());
}
}
}
}
} else {
if (formDataString != null) {
for (String part : parts) {
String[] kv = part.split("=");
if (kv != null) {
if (kv.length > 0) {
existingKeys.remove(kv[0] + ": fp");
}
if (kv.length == 2) {
// TODO how to handle arrays here?
String key = kv[0];
try {
String value = URLDecoder.decode(kv[1], "utf-8");
if (parameter.getName().equals(key)) {
JavaType jt = parameterClasses[i];
Class<?> cls = jt.getRawClass();
try {
o = validator.convertAndValidate(Arrays.asList(value), parameter, cls, definitions);
} catch (ConversionException e) {
missingParams.add(e.getError());
} catch (ValidationException e) {
missingParams.add(e.getValidationMessage());
}
}
} catch (UnsupportedEncodingException e) {
LOGGER.error("unable to decode value for " + key);
}
}
}
}
}
}
} else {
try {
String paramName = parameter.getName();
if ("query".equals(in)) {
existingKeys.remove(paramName + ": qp");
}
if ("path".equals(in)) {
existingKeys.remove(paramName + ": pp");
}
JavaType jt = parameterClasses[i];
Class<?> cls = jt.getRawClass();
if ("body".equals(in)) {
if (ctx.hasEntity()) {
BodyParameter body = (BodyParameter) parameter;
o = EntityProcessorFactory.readValue(ctx.getMediaType(), ctx.getEntityStream(), cls);
if (o != null) {
validate(o, body.getSchema(), SchemaValidator.Direction.INPUT);
}
} else if (parameter.getRequired()) {
ValidationException e = new ValidationException();
e.message(new ValidationMessage().message("The input body `" + paramName + "` is required"));
throw e;
}
}
if ("query".equals(in)) {
o = validator.convertAndValidate(uri.getQueryParameters().get(parameter.getName()), parameter, cls, definitions);
} else if ("path".equals(in)) {
o = validator.convertAndValidate(uri.getPathParameters().get(parameter.getName()), parameter, cls, definitions);
} else if ("header".equals(in)) {
o = validator.convertAndValidate(ctx.getHeaders().get(parameter.getName()), parameter, cls, definitions);
}
} catch (ConversionException e) {
missingParams.add(e.getError());
} catch (ValidationException e) {
missingParams.add(e.getValidationMessage());
}
}
} catch (NumberFormatException e) {
LOGGER.error("Couldn't find " + parameter.getName() + " (" + in + ") to " + parameterClasses[i], e);
}
args[i] = o;
i += 1;
}
if (existingKeys.size() > 0) {
LOGGER.debug("unexpected keys: " + existingKeys);
}
if (missingParams.size() > 0) {
StringBuilder builder = new StringBuilder();
builder.append("Input error");
if (missingParams.size() > 1) {
builder.append("s");
}
builder.append(": ");
int count = 0;
for (ValidationMessage message : missingParams) {
if (count > 0) {
builder.append(", ");
}
if (message != null && message.getMessage() != null) {
builder.append(message.getMessage());
} else {
builder.append("no additional input");
}
count += 1;
}
int statusCode = config.getInvalidRequestStatusCode();
ApiError error = new ApiError().code(statusCode).message(builder.toString());
throw new ApiException(error);
}
}
try {
if (method != null) {
LOGGER.info("calling method " + method + " on controller " + this.controller + " with args " + Arrays.toString(args));
try {
Object response = method.invoke(controller, args);
if (response instanceof ResponseContext) {
ResponseContext wrapper = (ResponseContext) response;
ResponseBuilder builder = Response.status(wrapper.getStatus());
// response headers
for (String key : wrapper.getHeaders().keySet()) {
List<String> v = wrapper.getHeaders().get(key);
if (v.size() == 1) {
builder.header(key, v.get(0));
} else {
builder.header(key, v);
}
}
// entity
if (wrapper.getEntity() != null) {
builder.entity(wrapper.getEntity());
// content type
if (wrapper.getContentType() != null) {
builder.type(wrapper.getContentType());
} else {
final ContextResolver<ContentTypeSelector> selector = providersProvider.get().getContextResolver(ContentTypeSelector.class, MediaType.WILDCARD_TYPE);
if (selector != null) {
selector.getContext(getClass()).apply(ctx.getAcceptableMediaTypes(), builder);
}
}
if (operation.getResponses() != null) {
String responseCode = String.valueOf(wrapper.getStatus());
io.swagger.models.Response responseSchema = operation.getResponses().get(responseCode);
if (responseSchema == null) {
// try default response schema
responseSchema = operation.getResponses().get("default");
}
if (responseSchema != null && responseSchema.getSchema() != null) {
validate(wrapper.getEntity(), responseSchema.getSchema(), SchemaValidator.Direction.OUTPUT);
} else {
LOGGER.debug("no response schema for code " + responseCode + " to validate against");
}
}
}
return builder.build();
}
return Response.ok().entity(response).build();
} catch (IllegalArgumentExceptionIllegalAccessException | InvocationTargetException | e) {
for (Throwable cause = e.getCause(); cause != null; ) {
if (cause instanceof ApiException) {
throw (ApiException) cause;
}
final Throwable next = cause.getCause();
cause = next == cause || next == null ? null : next;
}
throw new ApiException(ApiErrorUtils.createInternalError(), e);
}
}
Map<String, io.swagger.models.Response> responses = operation.getResponses();
if (responses != null) {
String[] keys = new String[responses.keySet().size()];
Arrays.sort(responses.keySet().toArray(keys));
int code = 0;
String defaultKey = null;
for (String key : keys) {
if (key.startsWith("2")) {
defaultKey = key;
code = Integer.parseInt(key);
break;
}
if ("default".equals(key)) {
defaultKey = key;
code = 200;
break;
}
if (key.startsWith("3")) {
// we use the 3xx responses as defaults
defaultKey = key;
code = Integer.parseInt(key);
}
}
if (defaultKey != null) {
ResponseBuilder builder = Response.status(code);
io.swagger.models.Response response = responses.get(defaultKey);
if (response.getHeaders() != null && response.getHeaders().size() > 0) {
for (String key : response.getHeaders().keySet()) {
Property headerProperty = response.getHeaders().get(key);
Object output = ExampleBuilder.fromProperty(headerProperty, definitions);
if (output instanceof ArrayExample) {
output = ((ArrayExample) output).asString();
} else if (output instanceof ObjectExample) {
LOGGER.debug("not serializing output example, only primitives or arrays of primitives are supported");
} else {
output = ((Example) output).asString();
}
builder.header(key, output);
}
}
Map<String, Object> examples = response.getExamples();
if (examples != null) {
for (MediaType mediaType : requestContext.getAcceptableMediaTypes()) {
for (String key : examples.keySet()) {
if (MediaType.valueOf(key).isCompatible(mediaType)) {
builder.entity(examples.get(key)).type(mediaType);
return builder.build();
}
}
}
}
Object output = ExampleBuilder.fromProperty(response.getSchema(), definitions);
if (output != null) {
ResponseContext resp = new ResponseContext().entity(output);
setContentType(requestContext, resp, operation);
builder.entity(output);
if (resp.getContentType() != null) {
// this comes from the operation itself
builder.type(resp.getContentType());
} else {
// get acceptable content types
List<EntityProcessor> processors = EntityProcessorFactory.getProcessors();
MediaType responseMediaType = null;
// take first compatible one
for (EntityProcessor processor : processors) {
if (responseMediaType != null) {
break;
}
for (MediaType mt : requestContext.getAcceptableMediaTypes()) {
LOGGER.debug("checking type " + mt.toString() + " against " + processor.getClass().getName());
if (processor.supports(mt)) {
builder.type(mt);
responseMediaType = mt;
break;
}
}
}
if (responseMediaType == null) {
// no match based on Accept header, use first processor in list
for (EntityProcessor processor : processors) {
List<MediaType> supportedTypes = processor.getSupportedMediaTypes();
if (supportedTypes.size() > 0) {
builder.type(supportedTypes.get(0));
break;
}
}
}
}
builder.entity(output);
}
return builder.build();
} else {
LOGGER.debug("no response type to map to, assume 200");
code = 200;
}
return Response.status(code).build();
}
return Response.ok().build();
} finally {
for (String key : inputStreams.keySet()) {
File file = inputStreams.get(key);
if (file != null) {
LOGGER.debug("deleting file " + file.getPath());
file.delete();
}
}
}
}Example 48
| Project: tika-master File: ISATabUtils.java View source code |
public static void parseStudy(InputStream stream, XHTMLContentHandler xhtml, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException {
TikaInputStream tis = TikaInputStream.get(stream);
// Automatically detect the character encoding
TikaConfig tikaConfig = context.get(TikaConfig.class);
if (tikaConfig == null) {
tikaConfig = TikaConfig.getDefaultConfig();
}
try (AutoDetectReader reader = new AutoDetectReader(new CloseShieldInputStream(tis), metadata, tikaConfig.getEncodingDetector());
CSVParser csvParser = new CSVParser(reader, CSVFormat.TDF)) {
Iterator<CSVRecord> iterator = csvParser.iterator();
xhtml.startElement("table");
xhtml.startElement("thead");
if (iterator.hasNext()) {
CSVRecord record = iterator.next();
for (int i = 0; i < record.size(); i++) {
xhtml.startElement("th");
xhtml.characters(record.get(i));
xhtml.endElement("th");
}
}
xhtml.endElement("thead");
xhtml.startElement("tbody");
while (iterator.hasNext()) {
CSVRecord record = iterator.next();
xhtml.startElement("tr");
for (int j = 0; j < record.size(); j++) {
xhtml.startElement("td");
xhtml.characters(record.get(j));
xhtml.endElement("td");
}
xhtml.endElement("tr");
}
xhtml.endElement("tbody");
xhtml.endElement("table");
}
}Example 49
| Project: TradeTrax-master File: Importer.java View source code |
public Importer onSuccessFromCsvForm() {
CSVParser parser = null;
parsed = new Vector<Stock>();
try {
parser = new CSVParser(new StringReader(rawcsv), CSVFormat.EXCEL.withHeader());
for (CSVRecord rec : parser) {
Stock stock = recordToStock(rec);
if (stock != null) {
parsed.add(stock);
}
}
} catch (Exception e) {
}
try {
parser.close();
} catch (Exception e) {
}
return this;
}Example 50
| Project: TundraCSV.java-master File: IDataCSVParser.java View source code |
/**
* Returns an IData representation of the CSV data in the given input stream.
*
* @param inputStream The input stream to be decoded.
* @param charset The character set to use.
* @return An IData representation of the given input stream data.
* @throws IOException If there is a problem reading from the stream.
*/
@Override
public IData decode(InputStream inputStream, Charset charset) throws IOException {
if (inputStream == null)
return null;
Reader reader = new InputStreamReader(inputStream, CharsetHelper.normalize(charset));
CSVFormat format = CSVFormat.DEFAULT.withHeader().withDelimiter(delimiter).withNullString("");
CSVParser parser = format.parse(reader);
Set<String> keys = parser.getHeaderMap().keySet();
List<IData> list = new ArrayList<IData>();
for (CSVRecord record : parser) {
IData document = IDataFactory.create();
IDataCursor cursor = document.getCursor();
for (String key : keys) {
if (record.isSet(key)) {
String value = record.get(key);
if (value != null)
IDataUtil.put(cursor, key, value);
}
}
cursor.destroy();
list.add(document);
}
IData output = IDataFactory.create();
IDataCursor cursor = output.getCursor();
IDataUtil.put(cursor, "recordWithNoID", list.toArray(new IData[list.size()]));
return output;
}Example 51
| Project: Web-Karma-master File: PatternReader.java View source code |
public static Map<String, Pattern> importPatterns(String inputDir, Integer length) {
Map<String, Pattern> patterns = new HashMap<String, Pattern>();
// CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator("\n").
// withIgnoreEmptyLines(true).
// withDelimiter(' ').
// withIgnoreSurroundingSpaces(true).
// withHeader();
CSVFormat csvFileFormat = CSVFormat.RFC4180.withHeader();
CSVParser csvParserTotalFrequency;
CSVParser csvParser;
File patternDir = new File(inputDir);
if (patternDir.exists()) {
File[] patternFiles = patternDir.listFiles();
for (File file : patternFiles) {
try {
if (file.isDirectory())
continue;
csvParser = CSVParser.parse(file, Charsets.UTF_8, csvFileFormat);
Map<String, Integer> headerMap = csvParser.getHeaderMap();
csvParserTotalFrequency = CSVParser.parse(file, Charsets.UTF_8, csvFileFormat);
int totalFrequency = getTotalFrequency(csvParserTotalFrequency, headerMap.get(COUNT_COLUMN));
for (CSVRecord record : csvParser) {
Pattern p = getPattern(record, headerMap, totalFrequency);
if (p != null) {
if (length != null) {
if (p.getLength() == length.intValue())
patterns.put(p.getId(), p);
} else {
patterns.put(p.getId(), p);
}
// System.out.println(p.getPrintStr());
} else {
logger.error("Error in reading the pattern: " + record.toString());
}
}
} catch (IOException e) {
logger.error("Error in reading the patterns from the file " + file.getAbsolutePath());
e.printStackTrace();
}
}
}
return patterns;
}Example 52
| Project: bw-calendar-engine-master File: ProcessCreate.java View source code |
private boolean createLocationsCsv() throws Throwable {
try {
open();
final String fileName = quotedVal();
if (fileName == null) {
error("Expected csv filename");
return false;
}
final File csvf = new File(fileName);
if (!csvf.exists()) {
error("File " + fileName + " does not exist");
return false;
}
final CSVParser parser = CSVFormat.DEFAULT.withHeader().withAllowMissingColumnNames().parse(new FileReader(csvf));
for (final CSVRecord rec : parser) {
final BwLocation loc = BwLocation.makeLocation();
String fld = rec.get("Address");
if (fld == null) {
error("address required for location");
continue;
}
loc.setAddressField(fld);
fld = rec.get("Room");
if (fld != null) {
loc.setRoomField(fld);
}
fld = rec.get("Code");
if (fld != null) {
loc.setSubField1(fld);
}
/* fld = rec.get("subField2");
if (fld != null) {
loc.setSubField1(fld);
}*/
fld = rec.get("Accessible");
if (fld != null) {
loc.setAccessible("Y".equals(fld));
}
fld = rec.get("Geouri");
if (fld != null) {
loc.setGeouri(fld);
}
fld = rec.get("Street");
if (fld != null) {
loc.setStreet(fld);
}
fld = rec.get("City");
if (fld != null) {
loc.setCity(fld);
}
fld = rec.get("State");
if (fld != null) {
loc.setState(fld);
}
fld = rec.get("Zip");
if (fld != null) {
loc.setZip(fld);
}
fld = rec.get("Link");
if (fld != null) {
loc.setLink(fld);
}
getSvci().getLocationsHandler().add(loc);
}
return true;
} finally {
close();
}
}Example 53
| Project: cloudsync-master File: Handler.java View source code |
private void readCSVStructure(final Path cacheFilePath) throws CloudsyncException {
final Map<String, Item> mapping = new HashMap<>();
mapping.put("", root);
try {
final Reader in = new FileReader(cacheFilePath.toFile());
final Iterable<CSVRecord> records = CSVFormat.EXCEL.parse(in);
for (final CSVRecord record : records) {
final Item item = Item.fromCSV(record);
final String childPath = Helper.trim(record.get(0), Item.SEPARATOR);
final String parentPath = childPath.length() == item.getName().length() ? "" : StringUtils.removeEnd(FilenameUtils.getPath(childPath), Item.SEPARATOR);
mapping.put(childPath, item);
// System.out.println(parentPath+":"+item.getName());
Item parent = mapping.get(parentPath);
item.setParent(parent);
parent.addChild(item);
}
} catch (final IOException e) {
throw new CloudsyncException("Can't read cache from file '" + cacheFilePath.toString() + "'", e);
}
}Example 54
| Project: orientdb-master File: OCSVExtractor.java View source code |
@Override
public boolean hasNext() {
if (recordIterator.hasNext()) {
CSVRecord csvRecord = recordIterator.next();
while (shouldSkipRecord(csvRecord) && recordIterator.hasNext()) {
csvRecord = recordIterator.next();
}
next = fetchNext(csvRecord);
return true;
}
return false;
}Example 55
| Project: pwm-master File: LocalDBUtility.java View source code |
private void importLocalDB(final InputStream inputStream, final Appendable out, final long totalBytes) throws PwmOperationalException, IOException {
this.prepareForImport();
importLineCounter = 0;
if (totalBytes > 0) {
writeStringToOut(out, "total bytes in localdb import source: " + totalBytes);
}
writeStringToOut(out, "beginning localdb import...");
final Instant startTime = Instant.now();
final TransactionSizeCalculator transactionCalculator = new TransactionSizeCalculator(new TransactionSizeCalculator.SettingsBuilder().setDurationGoal(new TimeDuration(100, TimeUnit.MILLISECONDS)).setMinTransactions(50).setMaxTransactions(5 * 1000).createSettings());
final Map<LocalDB.DB, Map<String, String>> transactionMap = new HashMap<>();
for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
transactionMap.put(loopDB, new TreeMap<>());
}
final CountingInputStream countingInputStream = new CountingInputStream(inputStream);
final EventRateMeter eventRateMeter = new EventRateMeter(TimeDuration.MINUTE);
final Timer statTimer = new Timer(true);
statTimer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
String output = "";
if (totalBytes > 0) {
final ProgressInfo progressInfo = new ProgressInfo(startTime, totalBytes, countingInputStream.getByteCount());
output += progressInfo.debugOutput();
} else {
output += "recordsImported=" + importLineCounter;
}
output += ", avgTransactionSize=" + transactionCalculator.getTransactionSize() + ", recordsPerMinute=" + eventRateMeter.readEventRate().setScale(2, BigDecimal.ROUND_DOWN);
writeStringToOut(out, output);
}
}, 30 * 1000, 30 * 1000);
Reader csvReader = null;
try {
csvReader = new InputStreamReader(new GZIPInputStream(countingInputStream, GZIP_BUFFER_SIZE), PwmConstants.DEFAULT_CHARSET);
for (final CSVRecord record : PwmConstants.DEFAULT_CSV_FORMAT.parse(csvReader)) {
importLineCounter++;
eventRateMeter.markEvents(1);
final String dbName_recordStr = record.get(0);
final LocalDB.DB db = JavaHelper.readEnumFromString(LocalDB.DB.class, null, dbName_recordStr);
final String key = record.get(1);
final String value = record.get(2);
if (db == null) {
writeStringToOut(out, "ignoring localdb import record #" + importLineCounter + ", invalid DB name '" + dbName_recordStr + "'");
} else {
transactionMap.get(db).put(key, value);
int cachedTransactions = 0;
for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
cachedTransactions += transactionMap.get(loopDB).size();
}
if (cachedTransactions >= transactionCalculator.getTransactionSize()) {
final long startTxnTime = System.currentTimeMillis();
for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
localDB.putAll(loopDB, transactionMap.get(loopDB));
transactionMap.get(loopDB).clear();
}
transactionCalculator.recordLastTransactionDuration(TimeDuration.fromCurrent(startTxnTime));
}
}
}
} finally {
LOGGER.trace("import process completed");
statTimer.cancel();
IOUtils.closeQuietly(csvReader);
IOUtils.closeQuietly(countingInputStream);
}
for (final LocalDB.DB loopDB : LocalDB.DB.values()) {
localDB.putAll(loopDB, transactionMap.get(loopDB));
transactionMap.get(loopDB).clear();
}
this.markImportComplete();
writeStringToOut(out, "restore complete, restored " + importLineCounter + " records in " + TimeDuration.fromCurrent(startTime).asLongString());
statTimer.cancel();
}Example 56
| Project: testsuite-nocoding-master File: CSVBasedURLActionDataListBuilder.java View source code |
@Override
public List<URLActionData> buildURLActionDataList() {
final List<URLActionData> actions = new ArrayList<URLActionData>();
@SuppressWarnings("unused") boolean incorrectLines = false;
try {
@SuppressWarnings("unchecked") final List<CSVRecord> records = (List<CSVRecord>) getOrParseData();
for (final CSVRecord csvRecord : records) {
if (csvRecord.isConsistent()) {
final URLActionData action = buildURLActionData(csvRecord);
actions.add(action);
} else {
XltLogger.runTimeLogger.error(new StringBuilder("Line at ").append(csvRecord.getRecordNumber()).append(" has not been correctly formatted. Line is ignored: ").append(csvRecord).toString());
incorrectLines = true;
}
}
} catch (final Exception e) {
throw new IllegalArgumentException(new StringBuilder("Failed to parse '").append(filePath).append("' as CSV data").toString(), e);
}
return actions;
}Example 57
| Project: camel-master File: CsvDataFormatTest.java View source code |
@Test
public void shouldHandleRecordConverter() {
CsvRecordConverter<String> converter = new CsvRecordConverter<String>() {
@Override
public String convertRecord(CSVRecord record) {
return record.toString();
}
};
CsvDataFormat dataFormat = new CsvDataFormat().setRecordConverter(converter);
// Properly saved
assertSame(CSVFormat.DEFAULT, dataFormat.getFormat());
assertSame(converter, dataFormat.getRecordConverter());
// Properly used (it doesn't modify the format)
assertEquals(CSVFormat.DEFAULT, dataFormat.getActiveFormat());
}Example 58
| Project: geowave-master File: SceneFeatureIterator.java View source code |
private void setupCsvToFeatureIterator(final File csvFile, final long startLine, final WRS2GeometryStore geometryStore, final Filter cqlFilter) throws FileNotFoundException, IOException {
parserFis = new FileInputStream(csvFile);
parserIsr = new InputStreamReader(parserFis, StringUtils.UTF8_CHAR_SET);
parser = new CSVParser(parserIsr, CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord());
final Iterator<CSVRecord> csvIterator = parser.iterator();
long startLineDecrementor = startLine;
// we skip the header, so only skip to start line 1
while ((startLineDecrementor > 1) && csvIterator.hasNext()) {
startLineDecrementor--;
csvIterator.next();
}
// wrap the iterator with a feature conversion and a filter (if
// provided)
iterator = Iterators.transform(csvIterator, new CSVToFeatureTransform(geometryStore, type));
if (cqlFilter != null) {
Filter actualFilter;
if (hasOtherProperties(cqlFilter)) {
final PropertyIgnoringFilterVisitor visitor = new PropertyIgnoringFilterVisitor(SCENE_ATTRIBUTES, type);
actualFilter = (Filter) cqlFilter.accept(visitor, null);
} else {
actualFilter = cqlFilter;
}
final CqlFilterPredicate filterPredicate = new CqlFilterPredicate(actualFilter);
iterator = Iterators.filter(iterator, filterPredicate);
}
}Example 59
| Project: limpet-master File: CsvParser.java View source code |
public List<IStoreItem> parse(String filePath) throws IOException {
final List<IStoreItem> res = new ArrayList<IStoreItem>();
final File inFile = new File(filePath);
final Reader in = new InputStreamReader(new FileInputStream(inFile), Charset.forName("UTF-8"));
final String fullFileName = inFile.getName();
final String fileName = filePrefix(fullFileName);
final Iterable<CSVRecord> records = CSVFormat.DEFAULT.parse(in);
boolean first = true;
// generate our list of importers
createImporters();
final DataImporter temporalDimensionless = new TemporalSeriesSupporter<Temporal.DimensionlessDouble>(Temporal.DimensionlessDouble.class, null, null);
final DataImporter temporalStrings = new TemporalStringImporter();
final DataImporter strings = new StringImporter();
final DataImporter dimensionless = new SeriesSupporter<NonTemporal.DimensionlessDouble>(NonTemporal.DimensionlessDouble.class, null, null);
// store one importer per column-set
List<DataImporter> importers = new ArrayList<DataImporter>();
// and store one series per column-set
List<ICollection> series = new ArrayList<ICollection>();
boolean isTime = false;
DateFormat customDateFormat = null;
for (CSVRecord record : records) {
if (first) {
first = false;
String time = record.get(0);
int ctr = 0;
if (time != null && time.toLowerCase().startsWith("time")) {
// is it plain time?
if (!time.toLowerCase().equals("time")) {
// ok, see if we have a time format string
if (time.contains("(") && time.contains(")")) {
// ok, extract the format string
String formatStr = time.substring(time.indexOf("(") + 1, time.indexOf(")"));
customDateFormat = new SimpleDateFormat(formatStr);
}
}
isTime = true;
ctr = 1;
} else {
ctr = 0;
}
while (ctr < record.size()) {
String nextVal = record.get(ctr);
// have a look at it.
int i1 = nextVal.indexOf("(");
String colName;
if (i1 > 0) {
// ok, we have units
colName = nextVal.substring(0, i1).trim();
} else {
// no, no units
colName = nextVal.trim();
}
// see if anybody can handle this name
boolean handled = false;
Iterator<DataImporter> cIter = _candidates.iterator();
while (cIter.hasNext()) {
DataImporter thisI = cIter.next();
if (thisI.handleName(colName)) {
importers.add(thisI);
series.add(thisI.create(fileName + "-" + thisI.nameFor(colName)));
handled = true;
ctr += thisI.numCols();
break;
}
}
if (!handled) {
int i2 = nextVal.indexOf(")");
if (i2 > 0 && i2 > i1 + 1) {
final String units = nextVal.substring(i1 + 1, i2).trim();
Iterator<DataImporter> cIter2 = _candidates.iterator();
while (cIter2.hasNext()) {
DataImporter thisI = cIter2.next();
if (thisI.handleUnits(units)) {
importers.add(thisI);
series.add(thisI.create(fileName + "-" + thisI.nameFor(colName)));
ctr += thisI.numCols();
handled = true;
break;
}
}
}
}
// have we managed it?
if (!handled) {
// ok, in that case we don't know. Let's introduce a deferred
// decision
// maker, so we can make a decision once we've read in some data
importers.add(new DeferredLoadSupporter(colName));
series.add(new ObjectCollection<String>("null"));
ctr += 1;
}
}
} else {
String firstRow = record.get(0);
long theTime = -1;
int thisCol = 0;
// ok, we're out of the first row
if (isTime) {
// ok, get the time field
try {
// do we have a custom date format
final DateFormat thisFormat;
if (customDateFormat != null) {
thisFormat = customDateFormat;
} else {
int len = firstRow.length();
if (len < 10) {
thisFormat = TIME_FORMAT;
} else {
thisFormat = DATE_FORMAT;
}
}
Date date = thisFormat.parse(firstRow);
theTime = date.getTime();
thisCol = 1;
} catch (ParseException e) {
e.printStackTrace();
}
} else {
// not temporal, use this field
thisCol = 0;
}
// now move through the other cols
int numImporters = importers.size();
for (int i = 0; i < numImporters; i++) {
DataImporter thisI = importers.get(i);
// ok, just check if this is a deferred importer
if (thisI instanceof DeferredLoadSupporter) {
DeferredLoadSupporter dl = (DeferredLoadSupporter) thisI;
String seriesName = dl.getName();
// ok, have a look at the next field
String nextVal = record.get(thisCol);
// is it numeric?
DataImporter importer = null;
// ok, treat it as string data
if (isTime) {
if (isNumeric(nextVal)) {
// ok, we've got dimensionless quantity data
importer = temporalDimensionless;
} else {
importer = temporalStrings;
}
} else {
if (isNumeric(nextVal)) {
// ok, we've got dimensionless quantity data
importer = dimensionless;
} else {
importer = strings;
}
}
if (importer != null) {
int index = importers.indexOf(dl);
importers.set(index, importer);
series.set(index, importer.create(fileName + "-" + seriesName));
thisI = importer;
}
}
ICollection thisS = series.get(i);
thisI.consume(thisS, theTime, thisCol, record);
thisCol += thisI.numCols();
}
}
}
// ok, store the series
if (series.size() > 1) {
StoreGroup target = new StoreGroup(fullFileName);
Iterator<ICollection> sIter = series.iterator();
while (sIter.hasNext()) {
ICollection coll = (ICollection) sIter.next();
target.add(coll);
}
res.add(target);
} else {
Iterator<ICollection> sIter = series.iterator();
while (sIter.hasNext()) {
ICollection coll = (ICollection) sIter.next();
res.add(coll);
}
}
return res;
}Example 60
| Project: opencast-master File: EventsLoader.java View source code |
private List<EventEntry> parseCSV(CSVParser csv) {
List<EventEntry> arrayList = new ArrayList<>();
for (CSVRecord record : csv) {
String title = record.get(0);
String description = StringUtils.trimToNull(record.get(1));
String series = StringUtils.trimToNull(record.get(2));
String seriesName = StringUtils.trimToNull(record.get(3));
Integer days = Integer.parseInt(record.get(4));
float signum = Math.signum(days);
DateTime now = DateTime.now();
if (signum > 0) {
now = now.plusDays(days);
} else if (signum < 0) {
now = now.minusDays(days * -1);
}
Integer duration = Integer.parseInt(record.get(5));
boolean archive = BooleanUtils.toBoolean(record.get(6));
String agent = StringUtils.trimToNull(record.get(7));
String source = StringUtils.trimToNull(record.get(8));
String contributor = StringUtils.trimToNull(record.get(9));
List<String> presenters = Arrays.asList(StringUtils.split(StringUtils.trimToEmpty(record.get(10)), ","));
EventEntry eventEntry = new EventEntry(title, now.toDate(), duration, archive, series, agent, source, contributor, description, seriesName, presenters);
arrayList.add(eventEntry);
}
return arrayList;
}Example 61
| Project: ProcessIsolatedTika-master File: ProcessIsolatedTika.java View source code |
/**
* Parse an inputstream and populate a Metadata object
* @param pInputStream stream to analyse
* @param pMetadata metadata object to populate
* @param pOutputStream output to write data to
* @return true if processed ok, false if execution was terminated
*/
public boolean parse(final InputStream pInputStream, final Metadata pMetadata) {
boolean ret = true;
if (!gRunner.isRunning()) {
gLogger.error("Tika-Server is not running");
return false;
}
final String TIKA_PATH = "/meta";
final String END_POINT = "http://" + TIKA_LOCAL_HOST + ":" + TIKA_SERVER_PORT;
gLogger.trace("Server: " + END_POINT + TIKA_PATH);
final String detectedType = pMetadata.get(Metadata.CONTENT_TYPE);
FutureTask<Integer> task = new FutureTask<Integer>(new Callable<Integer>() {
@Override
public Integer call() throws Exception {
gResponse = WebClient.create(END_POINT + TIKA_PATH).accept("text/csv").type(// give the parsers a hint
detectedType).put(// protect the stream from being closed
new CloseShieldInputStream(pInputStream));
return null;
}
});
Thread thread = new Thread(task);
thread.start();
try {
task.get(TIMEOUT_SECS * 1000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
gLogger.info("InterruptedException: " + e);
ret = false;
restart();
} catch (ExecutionException e) {
gLogger.info("ExecutionException: " + e);
ret = false;
restart();
} catch (TimeoutException e) {
gLogger.info("TimeoutException: " + e);
ret = false;
restart();
}
if (gResponse != null) {
if (gResponse.getStatus() == Status.UNSUPPORTED_MEDIA_TYPE.getStatusCode()) {
// the server may return HTTP 415 (unsupported) if it won't accept the mimetype
// handle this issue here
// add some text to the output
// FIXME: maybe change mimetype for a more visible error?
pMetadata.add("parseFailure415", "true");
gLogger.error("Parse Failure: HTTP 415 (format unsupported for parsing)");
} else {
if (gResponse.getEntity() instanceof InputStream) {
InputStream is = (InputStream) gResponse.getEntity();
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
try {
Iterable<CSVRecord> records = CSVFormat.DEFAULT.parse(reader);
for (CSVRecord record : records) {
pMetadata.add(record.get(0), record.get(1));
}
} catch (IOException e1) {
e1.printStackTrace();
ret = false;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
}
gLogger.info("Metadata entries: " + pMetadata.names().length);
return ret;
}Example 62
| Project: tabula-java-master File: TestSpreadsheetExtractor.java View source code |
@Test
public void testFindSpreadsheetsFromCells() throws IOException {
CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestSpreadsheetExtractor-CELLS.csv"), Charset.forName("utf-8"), CSVFormat.DEFAULT);
List<Cell> cells = new ArrayList<Cell>();
for (CSVRecord record : parse) {
cells.add(new Cell(Float.parseFloat(record.get(0)), Float.parseFloat(record.get(1)), Float.parseFloat(record.get(2)), Float.parseFloat(record.get(3))));
}
SpreadsheetExtractionAlgorithm se = new SpreadsheetExtractionAlgorithm();
List<Rectangle> expected = Arrays.asList(EXPECTED_RECTANGLES);
Collections.sort(expected);
List<Rectangle> foundRectangles = se.findSpreadsheetsFromCells(cells);
Collections.sort(foundRectangles);
assertTrue(foundRectangles.equals(expected));
}Example 63
| Project: tradelib-master File: Series.java View source code |
public static Series fromCsv(String path, boolean header, DateTimeFormatter dtf, LocalTime lt) throws Exception {
if (dtf == null) {
if (lt == null)
dtf = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
else
dtf = DateTimeFormatter.ISO_DATE;
}
// Parse and import the csv
CSVFormat csvFmt = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreSurroundingSpaces();
if (header)
csvFmt = csvFmt.withHeader();
CSVParser csv = csvFmt.parse(new BufferedReader(new FileReader(path)));
int ncols = -1;
Series result = null;
double[] values = null;
for (CSVRecord rec : csv.getRecords()) {
if (result == null) {
ncols = rec.size() - 1;
values = new double[ncols];
result = new Series(ncols);
}
for (int ii = 0; ii < ncols; ++ii) {
values[ii] = Double.parseDouble(rec.get(ii + 1));
}
LocalDateTime ldt;
if (lt != null) {
ldt = LocalDate.parse(rec.get(0), dtf).atTime(lt);
} else {
ldt = LocalDateTime.parse(rec.get(0), dtf);
}
result.append(ldt, values);
}
if (header) {
Map<String, Integer> headerMap = csv.getHeaderMap();
result.clearNames();
for (Map.Entry<String, Integer> me : headerMap.entrySet()) {
if (me.getValue() > 0)
result.setName(me.getKey(), me.getValue() - 1);
}
}
return result;
}Example 64
| Project: vitam-master File: RulesManagerFileImpl.java View source code |
@Override
public ArrayNode checkFile(InputStream rulesFileStream) throws IOException, ReferentialException, InvalidParseOperationException {
ParametersChecker.checkParameter(RULES_FILE_STREAM_IS_A_MANDATORY_PARAMETER, rulesFileStream);
if (!isCollectionEmptyForTenant()) {
throw new FileRulesException(String.format(RULES_COLLECTION_IS_NOT_EMPTY, getTenant()));
}
File csvFileReader = convertInputStreamToFile(rulesFileStream);
try (FileReader reader = new FileReader(csvFileReader)) {
@SuppressWarnings("resource") final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.withHeader());
final HashSet<String> ruleIdSet = new HashSet<>();
for (final CSVRecord record : parser) {
try {
if (checkRecords(record)) {
final String ruleId = record.get(RULEID);
final String ruleType = record.get(RULE_TYPE);
final String ruleValue = record.get(RULE_VALUE);
final String ruleDuration = record.get(RULE_DURATION);
final String ruleMeasurementValue = record.get(RULE_MEASUREMENT);
checkParametersNotEmpty(ruleId, ruleType, ruleValue, ruleDuration, ruleMeasurementValue);
checkRuleDurationIsInteger(ruleDuration);
if (ruleIdSet.contains(ruleId)) {
throw new FileRulesException(String.format(FILE_RULE_WITH_RULE_ID, ruleId));
}
ruleIdSet.add(ruleId);
if (!contains(ruleMeasurementValue)) {
throw new FileRulesException(String.format(NOT_SUPPORTED_VALUE, RULE_MEASUREMENT, ruleMeasurementValue));
}
}
} catch (final Exception e) {
throw new FileRulesException(INVALID_CSV_FILE + e.getMessage());
}
}
}
if (csvFileReader != null) {
final ArrayNode readRulesAsJson = RulesManagerParser.readObjectsFromCsvWriteAsArrayNode(csvFileReader);
csvFileReader.delete();
return readRulesAsJson;
}
/* this line is reached only if temporary file is null */
throw new FileRulesException(INVALID_CSV_FILE);
}Example 65
| Project: alma-toolkit-master File: TaskUpdateResourcePartners.java View source code |
public ConcurrentMap<String, Partner> getTepunaPartners() {
String prefix = "NLNZ";
String institutionCode = config.getProperty("ladd.institution.code");
ConcurrentMap<String, Partner> result = new ConcurrentHashMap<String, Partner>();
WebClient webClient = webClientProvider.get();
TextPage page = null;
try {
log.debug("tepuna url: {}", tepunaUrl);
page = webClient.getPage(tepunaUrl);
} catch (IOException e) {
log.error("unable to acquire page: {}", tepunaUrl);
return result;
}
log.debug("{}", page.getContent());
try (CSVParser parser = CSVParser.parse(page.getContent(), CSVFormat.DEFAULT.withHeader())) {
for (CSVRecord record : parser) {
String nuc = record.get(0);
if ("NUC symbol".equals(nuc) || institutionCode.equals(nuc)) {
log.debug("skipping nuc: {}", nuc);
continue;
}
nuc = prefix + ":" + nuc;
String org = record.get(2);
Partner partner = new Partner();
partner.setLink("https://api-ap.hosted.exlibrisgroup.com/almaws/v1/partners/" + nuc);
PartnerDetails partnerDetails = new PartnerDetails();
partner.setPartnerDetails(partnerDetails);
ProfileDetails profileDetails = new ProfileDetails();
partnerDetails.setProfileDetails(profileDetails);
profileDetails.setProfileType(ProfileType.ISO);
RequestExpiryType requestExpiryType = new RequestExpiryType();
requestExpiryType.setValue("INTEREST_DATE");
requestExpiryType.setDesc("Expire by interest date");
IsoDetails isoDetails = new IsoDetails();
profileDetails.setIsoDetails(isoDetails);
isoDetails.setAlternativeDocumentDelivery(false);
isoDetails.setIllServer(config.getProperty("alma.ill.server"));
isoDetails.setIllPort(Integer.parseInt(config.getProperty("alma.ill.port")));
isoDetails.setIsoSymbol(nuc);
isoDetails.setSendRequesterInformation(false);
isoDetails.setSharedBarcodes(true);
isoDetails.setRequestExpiryType(requestExpiryType);
SystemType systemType = new SystemType();
systemType.setValue("LADD");
systemType.setDesc("LADD");
LocateProfile locateProfile = new LocateProfile();
locateProfile.setValue("LADD");
locateProfile.setDesc("LADD Locate Profile");
partnerDetails.setStatus(Status.ACTIVE);
partnerDetails.setCode(nuc);
partnerDetails.setName(org);
partnerDetails.setSystemType(systemType);
partnerDetails.setAvgSupplyTime(4);
partnerDetails.setDeliveryDelay(4);
partnerDetails.setCurrency("AUD");
partnerDetails.setBorrowingSupported(true);
partnerDetails.setBorrowingWorkflow("LADD_Borrowing");
partnerDetails.setLendingSupported(true);
partnerDetails.setLendingWorkflow("LADD_Lending");
partnerDetails.setLocateProfile(locateProfile);
partnerDetails.setHoldingCode(nuc);
ContactInfo contactInfo = new ContactInfo();
partner.setContactInfo(contactInfo);
Addresses addresses = new Addresses();
contactInfo.setAddresses(addresses);
String s = record.get(5);
if (s == null || "".equals(s.trim()))
s = record.get(4);
if (s != null && !"".equals(s.trim())) {
Address address = getAddress(s);
address.setPreferred(true);
address.setAddressTypes(new AddressTypes());
address.getAddressTypes().getAddressType().add("ALL");
addresses.getAddress().add(address);
log.debug("nuc/address [{}]: {}", nuc, address);
}
Emails emails = new Emails();
contactInfo.setEmails(emails);
s = record.get(6);
if (s != null && !"".equals(s.trim())) {
Email email = new Email();
email.setEmailTypes(new EmailTypes());
email.setEmailAddress(s);
email.setPreferred(true);
email.setDescription("Primary Email Address");
email.getEmailTypes().getEmailType().add("ALL");
emails.getEmail().add(email);
log.debug("nuc/email1 [{}]: {}", nuc, email);
}
s = record.get(13);
if (s != null && !"".equals(s.trim())) {
Email email = new Email();
email.setEmailTypes(new EmailTypes());
email.setEmailAddress(s);
email.setPreferred(true);
String m = record.get(12);
if (m != null && !"".equals(m))
email.setDescription("Manager Email Address: " + m);
else
email.setDescription("Manager Email Address");
email.getEmailTypes().getEmailType().add("ALL");
emails.getEmail().add(email);
log.debug("nuc/email2 [{}]: {}", nuc, email);
}
Phones phones = new Phones();
contactInfo.setPhones(phones);
s = record.get(15);
if (s == null || "".equals(s.trim()))
s = record.get(7);
if (s != null && !"".equals(s.trim())) {
Phone phone = new Phone();
phone.setPhoneTypes(new PhoneTypes());
phone.setPhoneNumber(s);
phone.setPreferred(true);
phone.setPreferredSMS(false);
phone.getPhoneTypes().getPhoneType().add("ALL");
phones.getPhone().add(phone);
log.debug("nuc/phone [{}]: {}", nuc, phone);
}
Notes notes = new Notes();
partner.setNotes(notes);
result.put(partner.getPartnerDetails().getCode(), partner);
}
} catch (IOException ioe) {
log.error("unable to parse data: {}", tepunaUrl);
}
return result;
}Example 66
| Project: hapi-fhir-master File: TerminologyLoaderSvc.java View source code |
private void iterateOverZipFile(List<byte[]> theZipBytes, String fileNamePart, IRecordHandler handler, char theDelimiter, QuoteMode theQuoteMode) {
boolean found = false;
for (byte[] nextZipBytes : theZipBytes) {
ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new ByteArrayInputStream(nextZipBytes)));
try {
for (ZipEntry nextEntry; (nextEntry = zis.getNextEntry()) != null; ) {
ZippedFileInputStream inputStream = new ZippedFileInputStream(zis);
String nextFilename = nextEntry.getName();
if (nextFilename.contains(fileNamePart)) {
ourLog.info("Processing file {}", nextFilename);
found = true;
Reader reader = null;
CSVParser parsed = null;
try {
reader = new InputStreamReader(new BOMInputStream(zis), Charsets.UTF_8);
CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
if (theQuoteMode != null) {
format = format.withQuote('"').withQuoteMode(theQuoteMode);
}
parsed = new CSVParser(reader, format);
Iterator<CSVRecord> iter = parsed.iterator();
ourLog.debug("Header map: {}", parsed.getHeaderMap());
int count = 0;
int logIncrement = LOG_INCREMENT;
int nextLoggedCount = 0;
while (iter.hasNext()) {
CSVRecord nextRecord = iter.next();
handler.accept(nextRecord);
count++;
if (count >= nextLoggedCount) {
ourLog.info(" * Processed {} records in {}", count, nextFilename);
nextLoggedCount += logIncrement;
}
}
} catch (IOException e) {
throw new InternalErrorException(e);
}
}
}
} catch (IOException e) {
throw new InternalErrorException(e);
} finally {
IOUtils.closeQuietly(zis);
}
}
// This should always be true, but just in case we've introduced a bug...
Validate.isTrue(found);
}Example 67
| Project: rce-master File: DOESection.java View source code |
private boolean loadTableFromFile(String path) {
boolean success = true;
try {
File csvData = new File(path);
CSVParser parser = CSVParser.parse(FileUtils.readFileToString(csvData), CSVFormat.newFormat(';').withIgnoreSurroundingSpaces().withAllowMissingColumnNames().withRecordSeparator("\n"));
List<CSVRecord> records = parser.getRecords();
if (Character.isLetter((records.get(0).get(0).charAt(0)))) {
records.remove(0);
}
String[][] values = new String[records.size()][];
int count = 0;
for (CSVRecord record : records) {
int size = getOutputs().size();
values[count] = new String[size];
for (int i = 0; i < size; i++) {
if (record.size() > i) {
String number = record.get(i);
if (number.equals("null")) {
values[count][i] = "";
} else {
Matcher matcher = Pattern.compile("(\\+|-)?\\d+(,|.)?\\d*(e|E)?(\\+|-)?\\d*").matcher(number);
if (count == 0 && i == 0 && matcher.find()) {
number = matcher.group();
}
values[count][i] = number.replaceAll("\"", " ").replaceAll(",", ".");
}
} else {
values[count][i] = "";
}
}
count++;
}
ObjectMapper mapper = JsonUtils.getDefaultObjectMapper();
setProperty(DOEConstants.KEY_TABLE, mapper.writeValueAsString(values));
setProperty(DOEConstants.KEY_START_SAMPLE, "0");
setProperty(DOEConstants.KEY_END_SAMPLE, Integer.toString(count - 1));
setProperty(DOEConstants.KEY_RUN_NUMBER, Integer.toString(count));
setProperty(DOEConstants.KEY_METHOD, DOEConstants.DOE_ALGORITHM_CUSTOM_TABLE);
algorithmSelection.select(algorithmSelection.indexOf(DOEConstants.DOE_ALGORITHM_CUSTOM_TABLE));
refreshSection();
if (values[0] != null && values[0].length > getOutputs().size()) {
MessageDialog.openInformation(this.getComposite().getShell(), "Warning", "There are more values per run in the loaded table than outputs defined.");
}
} catch (IOException e) {
LOGGER.error(e);
success = false;
if (!success) {
MessageDialog.openInformation(this.getComposite().getShell(), ERROR, "Could not load table from file.\n\nReason: " + e.getMessage());
}
} catch (NumberFormatException e) {
LOGGER.error(e);
success = false;
if (!success) {
MessageDialog.openInformation(this.getComposite().getShell(), ERROR, "Could not load table because of a syntax error.\n\nReason: " + e.getMessage());
}
}
return success;
}Example 68
| Project: datumbox-framework-master File: Dataframe.java View source code |
/**
* It builds a Dataframe object from a CSV file; the first line of the provided
* CSV file must have a header with the column names.
*
* The method accepts the following arguments: A Reader object from where
* we will read the contents of the csv file. The name column of the
* response variable y. A map with the column names and their respective
* DataTypes. The char delimiter for the columns, the char for quotes and
* the string of the record/row separator. The Storage Configuration
* object.
*
* @param reader
* @param yVariable
* @param headerDataTypes
* @param delimiter
* @param quote
* @param recordSeparator
* @param skip
* @param limit
* @param configuration
* @return
*/
public static Dataframe parseCSVFile(Reader reader, String yVariable, LinkedHashMap<String, TypeInference.DataType> headerDataTypes, char delimiter, char quote, String recordSeparator, Long skip, Long limit, Configuration configuration) {
Logger logger = LoggerFactory.getLogger(Dataframe.Builder.class);
if (skip == null) {
skip = 0L;
}
if (limit == null) {
limit = Long.MAX_VALUE;
}
logger.info("Parsing CSV file");
if (!headerDataTypes.containsKey(yVariable)) {
logger.warn("WARNING: The file is missing the response variable column {}.", yVariable);
}
TypeInference.DataType yDataType = headerDataTypes.get(yVariable);
//copy header types
Map<String, TypeInference.DataType> xDataTypes = new HashMap<>(headerDataTypes);
//remove the response variable from xDataTypes
xDataTypes.remove(yVariable);
//use the private constructor to pass DataTypes directly and avoid updating them on the fly
Dataframe dataset = new Dataframe(configuration, yDataType, xDataTypes);
CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(delimiter).withQuote(quote).withRecordSeparator(recordSeparator);
try (final CSVParser parser = new CSVParser(reader, format)) {
ThreadMethods.throttledExecution(StreamMethods.enumerate(StreamMethods.stream(parser.spliterator(), false)).skip(skip).limit(limit), e -> {
Integer rId = e.getKey();
CSVRecord row = e.getValue();
if (!row.isConsistent()) {
logger.warn("WARNING: Skipping row {} because its size does not match the header size.", row.getRecordNumber());
} else {
Object y = null;
AssociativeArray xData = new AssociativeArray();
for (Map.Entry<String, TypeInference.DataType> entry : headerDataTypes.entrySet()) {
String column = entry.getKey();
TypeInference.DataType dataType = entry.getValue();
Object value = TypeInference.DataType.parse(row.get(column), dataType);
if (yVariable != null && yVariable.equals(column)) {
y = value;
} else {
xData.put(column, value);
}
}
Record r = new Record(xData, y);
dataset._unsafe_set(rId, r);
}
}, configuration.getConcurrencyConfiguration());
} catch (IOException ex) {
throw new RuntimeException(ex);
}
return dataset;
}Example 69
| Project: ofbiz-master File: PartyServices.java View source code |
public static Map<String, Object> importParty(DispatchContext dctx, Map<String, Object> context) {
Delegator delegator = dctx.getDelegator();
LocalDispatcher dispatcher = dctx.getDispatcher();
Locale locale = (Locale) context.get("locale");
GenericValue userLogin = (GenericValue) context.get("userLogin");
ByteBuffer fileBytes = (ByteBuffer) context.get("uploadedFile");
String encoding = System.getProperty("file.encoding");
String csvString = Charset.forName(encoding).decode(fileBytes).toString();
final BufferedReader csvReader = new BufferedReader(new StringReader(csvString));
CSVFormat fmt = CSVFormat.DEFAULT.withHeader();
List<String> errMsgs = new LinkedList<String>();
List<String> newErrMsgs = new LinkedList<String>();
// last partyId read from the csv file
String lastPartyId = null;
// current partyId from the csv file
String currentPartyId = null;
// new to create/update partyId in the system
String newPartyId = null;
String newCompanyPartyId = null;
int partiesCreated = 0;
Map<String, Object> result = null;
String newContactMechId = null;
String currentContactMechTypeId = null;
String lastAddress1 = null;
String lastAddress2 = null;
String lastCity = null;
String lastCountryGeoId = null;
String lastEmailAddress = null;
String lastCountryCode = null;
String lastAreaCode = null;
String lastContactNumber = null;
String lastContactMechPurposeTypeId = null;
String currentContactMechPurposeTypeId = null;
// when modify party, contact mech not added again
Boolean addParty = false;
if (fileBytes == null) {
return ServiceUtil.returnError(UtilProperties.getMessage(resourceError, "PartyUploadedFileDataNotFound", locale));
}
try {
for (final CSVRecord rec : fmt.parse(csvReader)) {
if (UtilValidate.isNotEmpty(rec.get("partyId"))) {
currentPartyId = rec.get("partyId");
}
if (lastPartyId == null || !currentPartyId.equals(lastPartyId)) {
newPartyId = null;
currentContactMechPurposeTypeId = null;
lastAddress1 = null;
lastAddress2 = null;
lastCity = null;
lastCountryGeoId = null;
lastEmailAddress = null;
lastCountryCode = null;
lastAreaCode = null;
lastContactNumber = null;
// party validation
List<GenericValue> currencyCheck = EntityQuery.use(delegator).from("Uom").where("abbreviation", rec.get("preferredCurrencyUomId"), "uomTypeId", "CURRENCY_MEASURE").queryList();
if (UtilValidate.isNotEmpty(rec.get("preferredCurrencyUomId")) && currencyCheck.size() == 0) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Currency code not found for: " + rec.get("preferredCurrencyUomId"));
}
if (UtilValidate.isEmpty(rec.get("roleTypeId"))) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory roletype is missing, possible values: CUSTOMER, SUPPLIER, EMPLOYEE and more....");
} else if (EntityQuery.use(delegator).from("RoleType").where("roleTypeId", rec.get("roleTypeId")).queryOne() == null) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + ": RoletypeId is not valid: " + rec.get("roleTypeId"));
}
if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && EntityQuery.use(delegator).from("ContactMechType").where("contactMechTypeId", rec.get("contactMechTypeId")).cache().queryOne() == null) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + " contactMechTypeId code not found for: " + rec.get("contactMechTypeId"));
}
if (UtilValidate.isNotEmpty(rec.get("contactMechPurposeTypeId")) && EntityQuery.use(delegator).from("ContactMechPurposeType").where("contactMechPurposeTypeId", rec.get("contactMechPurposeTypeId")).cache().queryOne() == null) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "contactMechPurposeTypeId code not found for: " + rec.get("contactMechPurposeTypeId"));
}
if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "POSTAL_ADDRESS".equals(rec.get("contactMechTypeId"))) {
if (UtilValidate.isEmpty(rec.get("countryGeoId"))) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Country code missing");
} else {
List<GenericValue> countryCheck = EntityQuery.use(delegator).from("Geo").where("geoTypeId", "COUNTRY", "abbreviation", rec.get("countryGeoId")).queryList();
if (countryCheck.size() == 0) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid Country code: " + rec.get("countryGeoId"));
}
}
if (UtilValidate.isEmpty(rec.get("city"))) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + "City name is missing");
}
if (UtilValidate.isNotEmpty(rec.get("stateProvinceGeoId"))) {
List<GenericValue> stateCheck = EntityQuery.use(delegator).from("Geo").where("geoTypeId", "STATE", "abbreviation", rec.get("stateProvinceGeoId")).queryList();
if (stateCheck.size() == 0) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid stateProvinceGeoId code: " + rec.get("countryGeoId"));
}
}
}
if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "TELECOM_NUMBER".equals(rec.get("contactMechTypeId"))) {
if (UtilValidate.isEmpty(rec.get("telAreaCode")) && UtilValidate.isEmpty(rec.get("telAreaCode"))) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " telephone number missing");
}
}
if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "EMAIL_ADDRESS".equals(rec.get("contactMechTypeId"))) {
if (UtilValidate.isEmpty(rec.get("emailAddress"))) {
newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " email address missing");
}
}
if (errMsgs.size() == 0) {
List<GenericValue> partyCheck = EntityQuery.use(delegator).from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")).queryList();
addParty = partyCheck.size() == 0;
if (!addParty) {
// update party
newPartyId = EntityUtil.getFirst(partyCheck).getString("partyId");
if (UtilValidate.isNotEmpty(rec.get("groupName"))) {
Map<String, Object> partyGroup = UtilMisc.toMap("partyId", newPartyId, "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin);
result = dispatcher.runSync("updatePartyGroup", partyGroup);
} else {
// person
Map<String, Object> person = UtilMisc.toMap("partyId", newPartyId, "firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "userLogin", userLogin);
result = dispatcher.runSync("updatePerson", person);
}
} else {
// create new party
if (UtilValidate.isNotEmpty(rec.get("groupName"))) {
Map<String, Object> partyGroup = UtilMisc.toMap("preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin, "statusId", "PARTY_ENABLED");
result = dispatcher.runSync("createPartyGroup", partyGroup);
} else {
// person
Map<String, Object> person = UtilMisc.toMap("firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "statusId", "PARTY_ENABLED", "userLogin", userLogin);
result = dispatcher.runSync("createPerson", person);
}
newPartyId = (String) result.get("partyId");
Map<String, Object> partyIdentification = UtilMisc.toMap("partyId", newPartyId, "partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId"), "userLogin", userLogin);
result = dispatcher.runSync("createPartyIdentification", partyIdentification);
Map<String, Object> partyRole = UtilMisc.toMap("partyId", newPartyId, "roleTypeId", rec.get("roleTypeId"), "userLogin", userLogin);
dispatcher.runSync("createPartyRole", partyRole);
if (UtilValidate.isNotEmpty(rec.get("companyPartyId"))) {
List<GenericValue> companyCheck = EntityQuery.use(delegator).from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")).queryList();
if (companyCheck.size() == 0) {
// update party group
// company does not exist so create
Map<String, Object> companyPartyGroup = UtilMisc.toMap("partyId", newCompanyPartyId, "statusId", "PARTY_ENABLED", "userLogin", userLogin);
result = dispatcher.runSync("createPartyGroup", companyPartyGroup);
newCompanyPartyId = (String) result.get("partyId");
} else {
newCompanyPartyId = EntityUtil.getFirst(companyCheck).getString("partyId");
}
Map<String, Object> companyRole = UtilMisc.toMap("partyId", newCompanyPartyId, "roleTypeId", "ACCOUNT", "userLogin", userLogin);
dispatcher.runSync("createPartyRole", companyRole);
// company exist, so create link
Map<String, Object> partyRelationship = UtilMisc.toMap("partyIdTo", newPartyId, "partyIdFrom", newCompanyPartyId, "roleTypeIdFrom", "ACCOUNT", "partyRelationshipTypeId", "EMPLOYMENT", "userLogin", userLogin);
result = dispatcher.runSync("createPartyRelationship", partyRelationship);
}
}
Debug.logInfo(" =========================================================party created id: " + newPartyId, module);
partiesCreated++;
} else {
errMsgs.addAll(newErrMsgs);
newErrMsgs = new LinkedList<String>();
}
}
currentContactMechTypeId = rec.get("contactMechTypeId");
currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId");
// party correctly created (not updated) and contactMechtype provided?
if (newPartyId != null && addParty && UtilValidate.isNotEmpty(currentContactMechTypeId)) {
// fill maps and check changes
Map<String, Object> emailAddress = UtilMisc.toMap("contactMechTypeId", "EMAIL_ADDRESS", "userLogin", userLogin);
Boolean emailAddressChanged = false;
if ("EMAIL_ADDRESS".equals(currentContactMechTypeId)) {
emailAddress.put("infoString", rec.get("emailAddress"));
emailAddressChanged = lastEmailAddress == null || !lastEmailAddress.equals(rec.get("emailAddress"));
lastEmailAddress = rec.get("emailAddress");
}
// casting is here necessary for some compiler versions
Map<String, Object> postalAddress = UtilMisc.toMap("userLogin", (Object) userLogin);
Boolean postalAddressChanged = false;
if ("POSTAL_ADDRESS".equals(currentContactMechTypeId)) {
postalAddress.put("address1", rec.get("address1"));
postalAddress.put("address2", rec.get("address2"));
postalAddress.put("city", rec.get("city"));
postalAddress.put("stateProvinceGeoId", rec.get("stateProvinceGeoId"));
postalAddress.put("countryGeoId", rec.get("countryGeoId"));
postalAddress.put("postalCode", rec.get("postalCode"));
postalAddressChanged = lastAddress1 == null || !lastAddress1.equals(postalAddress.get("address1")) || lastAddress2 == null || !lastAddress2.equals(postalAddress.get("address2")) || lastCity == null || !lastCity.equals(postalAddress.get("city")) || lastCountryGeoId == null || !lastCountryGeoId.equals(postalAddress.get("countryGeoId"));
lastAddress1 = (String) postalAddress.get("address1");
lastAddress2 = (String) postalAddress.get("address2");
lastCity = (String) postalAddress.get("city");
lastCountryGeoId = (String) postalAddress.get("countryGeoId");
}
// casting is here necessary for some compiler versions
Map<String, Object> telecomNumber = UtilMisc.toMap("userLogin", (Object) userLogin);
Boolean telecomNumberChanged = false;
if ("TELECOM_NUMBER".equals(currentContactMechTypeId)) {
telecomNumber.put("countryCode", rec.get("telCountryCode"));
telecomNumber.put("areaCode", rec.get("telAreaCode"));
telecomNumber.put("contactNumber", rec.get("telContactNumber"));
telecomNumberChanged = lastCountryCode == null || !lastCountryCode.equals(telecomNumber.get("countryCode")) || lastAreaCode == null || !lastAreaCode.equals(telecomNumber.get("areaCode")) || lastContactNumber == null || !lastContactNumber.equals(telecomNumber.get("contactNumber"));
lastCountryCode = (String) telecomNumber.get("countryCode");
lastAreaCode = (String) telecomNumber.get("areaCode");
lastContactNumber = (String) telecomNumber.get("contactNumber");
}
Map<String, Object> partyContactMechPurpose = UtilMisc.toMap("partyId", newPartyId, "userLogin", userLogin);
Boolean partyContactMechPurposeChanged = false;
currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId");
if (currentContactMechPurposeTypeId != null && ("TELECOM_NUMBER".equals(currentContactMechTypeId) || "POSTAL_ADDRESS".equals(currentContactMechTypeId) || "EMAIL_ADDRESS".equals(currentContactMechTypeId))) {
partyContactMechPurpose.put("contactMechPurposeTypeId", currentContactMechPurposeTypeId);
partyContactMechPurposeChanged = (lastContactMechPurposeTypeId == null || !lastContactMechPurposeTypeId.equals(currentContactMechPurposeTypeId)) && !telecomNumberChanged && !postalAddressChanged && !emailAddressChanged;
Debug.logInfo("===================================last:" + lastContactMechPurposeTypeId + " current: " + currentContactMechPurposeTypeId + " t :" + telecomNumberChanged + " p: " + postalAddressChanged + " e: " + emailAddressChanged + " result: " + partyContactMechPurposeChanged, module);
}
lastContactMechPurposeTypeId = currentContactMechPurposeTypeId;
// update
if (errMsgs.size() == 0) {
if (postalAddressChanged) {
result = dispatcher.runSync("createPostalAddress", postalAddress);
newContactMechId = (String) result.get("contactMechId");
if (currentContactMechPurposeTypeId == null) {
currentContactMechPurposeTypeId = "GENERAL_LOCATION";
}
dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
}
if (telecomNumberChanged) {
result = dispatcher.runSync("createTelecomNumber", telecomNumber);
newContactMechId = (String) result.get("contactMechId");
if (currentContactMechPurposeTypeId == null) {
currentContactMechPurposeTypeId = "PHONE_WORK";
}
dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
}
if (emailAddressChanged) {
result = dispatcher.runSync("createContactMech", emailAddress);
newContactMechId = (String) result.get("contactMechId");
if (currentContactMechPurposeTypeId == null) {
currentContactMechPurposeTypeId = "PRIMARY_EMAIL";
}
dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
}
if (partyContactMechPurposeChanged) {
partyContactMechPurpose.put("contactMechId", newContactMechId);
result = dispatcher.runSync("createPartyContactMechPurpose", partyContactMechPurpose);
}
lastPartyId = currentPartyId;
errMsgs.addAll(newErrMsgs);
newErrMsgs = new LinkedList<String>();
}
}
}
} catch (GenericServiceException e) {
Debug.logError(e, module);
return ServiceUtil.returnError(e.getMessage());
} catch (GenericEntityException e) {
Debug.logError(e, module);
return ServiceUtil.returnError(e.getMessage());
} catch (IOException e) {
Debug.logError(e, module);
return ServiceUtil.returnError(e.getMessage());
}
if (errMsgs.size() > 0) {
return ServiceUtil.returnError(errMsgs);
}
result = ServiceUtil.returnSuccess(UtilProperties.getMessage(resource, "PartyNewPartiesCreated", UtilMisc.toMap("partiesCreated", partiesCreated), locale));
return result;
}Example 70
| Project: jena-master File: CSVParser.java View source code |
private static List<String> recordToList(CSVRecord record) {
return Iter.toList(record.iterator());
}Example 71
| Project: easybatch-framework-master File: ApacheCommonCsvRecordMapper.java View source code |
@Override
public Record<P> processRecord(final StringRecord record) throws Exception {
String payload = record.getPayload();
CSVParser csvParser = csvFormat.parse(new StringReader(payload));
CSVRecord csvRecord = csvParser.iterator().next();
csvParser.close();
return new GenericRecord<>(record.getHeader(), objectMapper.mapObject(csvRecord.toMap()));
}Example 72
| Project: open-data-service-master File: CsvSourceAdapter.java View source code |
private ObjectNode createObject(CSVRecord keys, CSVRecord values) { ObjectNode node = new ObjectNode(JsonNodeFactory.instance); for (int i = 0; i < keys.size(); ++i) { node.put(keys.get(i), values.get(i)); } return node; }
Example 73
| Project: extract-master File: LoadQueueTask.java View source code |
/**
* Load a dump file from the given input stream into a queue.
*
* @param queue the queue to load the dump into
* @param input the input stream to load the dump from
* @throws IOException if the dump could not be loaded
*/
private void loadFromCSV(final DocumentFactory factory, final DocumentQueue queue, final InputStream input) throws IOException {
final String pathField = options.get("pathField").value().orElse("path");
for (CSVRecord record : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(input))) {
queue.add(factory.create(Paths.get(record.get(pathField))));
}
}