Java Examples for org.apache.commons.csv.CSVParser
The following java examples will help you to understand the usage of org.apache.commons.csv.CSVParser. These source code samples are taken from different open source projects.
Example 1
| Project: DoSeR-master File: WordcountMapper.java View source code |
@Override
public void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(value.getBytes());
int n = value.getLength();
byte[] bytes = new byte[n];
inputStream.read(bytes, 0, n);
CSVParser parser = CSVParser.parse(new String(bytes), CSVFormat.DEFAULT);
try {
for (CSVRecord csvRecord : parser) {
for (String string : csvRecord) {
word.set(string);
context.write(word, one);
Counter counter = context.getCounter(CountersEnum.class.getName(), CountersEnum.INPUT_WORDS.toString());
counter.increment(1);
}
}
} catch (Exception e) {
}
}Example 2
| Project: jat-master File: CSVFileOutputReader.java View source code |
@Override
public List<JATETerm> read(String file) throws IOException {
CSVParser parser = new CSVParser(new FileReader(file), format);
List<JATETerm> out = new ArrayList<>();
for (CSVRecord rec : parser.getRecords()) {
out.add(new JATETerm(rec.get(0).trim(), Double.valueOf(rec.get(1).trim())));
}
Collections.sort(out);
return out;
}Example 3
| Project: tabula-java-master File: TestSpreadsheetExtractor.java View source code |
@Test
public void testFindSpreadsheetsFromCells() throws IOException {
CSVParser parse = org.apache.commons.csv.CSVParser.parse(new File("src/test/resources/technology/tabula/csv/TestSpreadsheetExtractor-CELLS.csv"), Charset.forName("utf-8"), CSVFormat.DEFAULT);
List<Cell> cells = new ArrayList<Cell>();
for (CSVRecord record : parse) {
cells.add(new Cell(Float.parseFloat(record.get(0)), Float.parseFloat(record.get(1)), Float.parseFloat(record.get(2)), Float.parseFloat(record.get(3))));
}
SpreadsheetExtractionAlgorithm se = new SpreadsheetExtractionAlgorithm();
List<Rectangle> expected = Arrays.asList(EXPECTED_RECTANGLES);
Collections.sort(expected);
List<Rectangle> foundRectangles = se.findSpreadsheetsFromCells(cells);
Collections.sort(foundRectangles);
assertTrue(foundRectangles.equals(expected));
}Example 4
| Project: hackday-master File: ExtratorKhan.java View source code |
public static void main(String[] args) throws IOException {
File csvData = new File("khan.csv");
CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8, CSVFormat.RFC4180);
PrintStream out = new PrintStream("khan.sql");
int registro = 1;
for (CSVRecord csvRecord : parser) {
if (registro++ == 1)
continue;
out.print("insert into khan (");
out.print("rede,escola,turma,login,aluno,com_dificuldade,precisa_praticar,praticado,nivel1,nivel2,dominado,pontos,exerciseminutes,videominutes,totalminutes,semana");
out.print(") values (");
out.print("'" + limpaTexto(csvRecord.get(0)) + "',");
out.print("'" + limpaTexto(csvRecord.get(1)) + "',");
out.print("'" + limpaTexto(csvRecord.get(2).replace("'", "")) + "',");
out.print("'" + limpaTexto(csvRecord.get(3)) + "',");
out.print("'" + limpaTexto(csvRecord.get(4)) + "',");
out.print("'" + limpaNumero(csvRecord.get(5)) + "',");
out.print("'" + limpaNumero(csvRecord.get(6)) + "',");
out.print("'" + limpaNumero(csvRecord.get(7)) + "',");
out.print("'" + limpaNumero(csvRecord.get(8)) + "',");
out.print("'" + limpaNumero(csvRecord.get(9)) + "',");
out.print("'" + limpaNumero(csvRecord.get(10)) + "',");
out.print("'" + limpaNumero(csvRecord.get(11)) + "',");
out.print("'" + limpaNumero(csvRecord.get(12)) + "',");
out.print("'" + limpaNumero(csvRecord.get(13)) + "',");
out.print("'" + limpaNumero(csvRecord.get(14)) + "',");
out.print("'" + csvRecord.get(15) + "')");
out.print(";");
out.println();
}
out.close();
}Example 5
| Project: janusz-master File: CommandInvoker.java View source code |
public String invoke(String sender, String messageContent) {
try {
String command = messageContent;
List<String> commandArgs = new ArrayList<>();
if (messageContent.contains(" ")) {
command = substringBefore(messageContent, " ");
String args = substringAfter(messageContent, " ");
CSVParser csvRecords = CSVParser.parse(args, CSVFormat.newFormat(' ').withQuote('"'));
commandArgs = Lists.newArrayList(csvRecords.iterator().next().iterator());
commandArgs = commandArgs.stream().map(( arg) -> arg.startsWith("$") ? store.get(sender, substringAfter(arg, "$"), String.class) : arg).collect(Collectors.toList());
}
return commands.get(command).invoke(sender, commandArgs);
} catch (Exception ex) {
throw new JanuszException(ex);
}
}Example 6
| Project: sw360portal-master File: UserPortlet.java View source code |
private List<UserCSV> getUsersFromRequest(PortletRequest request, String fileUploadFormId) throws IOException, TException {
final UploadPortletRequest uploadPortletRequest = PortalUtil.getUploadPortletRequest(request);
final InputStream stream = uploadPortletRequest.getFileAsStream(fileUploadFormId);
Reader reader = new InputStreamReader(stream);
CSVFormat format = CommonUtils.sw360CsvFormat;
CSVParser parser = new CSVParser(reader, format);
List<CSVRecord> records;
records = parser.getRecords();
if (records.size() > 0) {
// Remove header
records.remove(0);
}
return getUsersFromCSV(records);
}Example 7
| Project: act-master File: Runner.java View source code |
public static void main(String[] args) throws Exception {
BufferedReader reader = new BufferedReader(new FileReader(args[0]));
BufferedWriter writer = new BufferedWriter(new FileWriter(args[1]));
try {
Oscar oscar = new Oscar();
String line = null;
/* NOTE: this is exactly the wrong way to write a TSV reader. Caveat emptor.
* See http://tburette.github.io/blog/2014/05/25/so-you-want-to-write-your-own-CSV-code/
* and then use org.apache.commons.csv.CSVParser instead.
*/
while ((line = reader.readLine()) != null) {
// TSV means split on tabs! Nothing else will do.
List<String> fields = Arrays.asList(line.split("\t"));
// Choke if our invariants aren't satisfied. We expect ever line to have a name and an InChI.
if (fields.size() != 2) {
throw new RuntimeException(String.format("Found malformed line (all lines must have two fields: %s", line));
}
String name = fields.get(1);
List<ResolvedNamedEntity> entities = oscar.findAndResolveNamedEntities(name);
System.out.println("**********");
System.out.println("Name: " + name);
List<String> outputFields = new ArrayList<>(fields.size() + 1);
outputFields.addAll(fields);
if (entities.size() == 0) {
System.out.println("No match");
outputFields.add("noMatch");
} else if (entities.size() == 1) {
ResolvedNamedEntity entity = entities.get(0);
NamedEntity ne = entity.getNamedEntity();
if (ne.getStart() != 0 || ne.getEnd() != name.length()) {
System.out.println("Partial match");
printEntity(entity);
outputFields.add("partialMatch");
} else {
System.out.println("Exact match");
printEntity(entity);
outputFields.add("exactMatch");
List<ChemicalStructure> structures = entity.getChemicalStructures(FormatType.STD_INCHI);
for (ChemicalStructure s : structures) {
outputFields.add(s.getValue());
}
}
} else {
// Multiple matches found!
System.out.println("Multiple matches");
for (ResolvedNamedEntity e : entities) {
printEntity(e);
}
outputFields.add("multipleMatches");
}
writer.write(String.join("\t", outputFields));
writer.newLine();
}
} finally {
writer.flush();
writer.close();
}
}Example 8
| Project: datasalt-utils-master File: CSVMapper.java View source code |
@Override
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
id.set(prefix + key.toString());
if (key.get() == 0 && skipHeader) {
return;
}
CSVParser parser = new CSVParser(new StringReader(value.toString()), strategy);
String[] fields = null;
try {
fields = parser.getLine();
} catch (Exception e) {
LOG.warn("invalid line: '" + value + "': " + e.toString());
return;
}
if (fieldNames != null && fields.length != fieldNames.length) {
LOG.warn("header.len != line.len: " + value.toString());
return;
}
res.clear();
for (int i = 0; i < fields.length; i++) {
if (fieldNames != null) {
res.put(new Text(fieldNames[i]), new Text(fields[i]));
} else {
res.put(new Text("f" + i), new Text(fields[i]));
}
}
context.write(id, res);
}Example 9
| Project: find-master File: PlatformDataExportServiceIT.java View source code |
@Test
public void exportToCsv() throws E, IOException {
final R queryRequest = queryRequestBuilderFactory.getObject().queryRestrictions(testUtils.buildQueryRestrictions()).queryType(QueryRequest.QueryType.MODIFIED).build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
exportService.exportQueryResults(outputStream, queryRequest, ExportFormat.CSV, Collections.emptyList(), 1001L);
final String output = outputStream.toString();
assertNotNull(output);
try (final CSVParser csvParser = CSVParser.parse(output, CSVFormat.EXCEL)) {
final List<CSVRecord> records = csvParser.getRecords();
assertThat(records, not(empty()));
final CSVRecord headerRecord = records.get(0);
// byte-order mark may get in the way
assertThat(headerRecord.get(0), endsWith("Reference"));
assertEquals("Database", headerRecord.get(1));
final CSVRecord firstDataRecord = records.get(1);
final String firstDataRecordReference = firstDataRecord.get(0);
assertNotNull(firstDataRecordReference);
assertFalse(firstDataRecordReference.trim().isEmpty());
final String firstDataRecordDatabase = firstDataRecord.get(1);
assertFalse(firstDataRecordDatabase.trim().isEmpty());
}
}Example 10
| Project: ml-master File: StringSplitFn.java View source code |
@Override
public void process(String line, Emitter<Record> emitter) {
if (line == null || line.isEmpty()) {
return;
}
try {
String[] pieces = new CSVParser(new StringReader(line), csvStrategy).getLine();
if (pieces != null) {
emitter.emit(new CSVRecord(pieces));
}
} catch (IOException e) {
throw new CrunchRuntimeException(e);
}
}Example 11
| Project: uadetector-master File: OperatingSystemSampleReader.java View source code |
private static List<OperatingSystemSample> read(final String file) {
final InputStream stream = UserAgentStringParserIntegrationTest.class.getClassLoader().getResourceAsStream(file);
CSVParser csvParser = null;
try {
csvParser = new CSVParser(new InputStreamReader(stream, CHARSET));
} catch (final UnsupportedEncodingException e) {
LOG.warn(e.getLocalizedMessage(), e);
}
final List<OperatingSystemSample> examples = new ArrayList<OperatingSystemSample>();
if (csvParser != null) {
String[] line = null;
int i = 0;
do {
try {
line = csvParser.getLine();
} catch (final IOException e) {
line = null;
LOG.warn(e.getLocalizedMessage(), e);
}
if (line != null) {
i++;
if (line.length == 4) {
final OperatingSystemFamily family = OperatingSystemFamily.valueOf(line[0]);
final String name = line[1];
final VersionNumber version = VersionParser.parseVersion(line[2]);
final String userAgent = line[3];
examples.add(new OperatingSystemSample(family, name, version, userAgent));
} else {
LOG.warn("Can not read operating system example " + i + ", there are too few fields: " + line.length);
}
}
} while (line != null);
}
return examples;
}Example 12
| Project: android-speedtest-mapper-master File: CsvDataParser.java View source code |
/**
* Parses CSV data
*
* @param csvHeader Header items for csv records
* @param csvData
* @return
*/
public static List<SpeedTestRecord> parseCsvData(String csvHeader, String csvData) {
Reader in = new StringReader(getCsvData(csvHeader, csvData));
try {
CSVParser parser = new CSVParser(in, CSVFormat.DEFAULT.toBuilder().withHeader().build());
// get the parsed records
List<CSVRecord> list = parser.getRecords();
// create a list to convert data into SpeedTestRecord model
List<SpeedTestRecord> speedTestRecords = new ArrayList<SpeedTestRecord>();
for (CSVRecord csvRecord : list) {
speedTestRecords.add(new SpeedTestRecord(csvRecord));
}
return speedTestRecords;
} catch (IOException e) {
Tracer.error(LOG_TAG, e);
}
// when no data, send empty list
return Collections.emptyList();
}Example 13
| Project: Decision-master File: DataFlowFromCsvMain.java View source code |
public static void main(String[] args) throws IOException, NumberFormatException, InterruptedException {
if (args.length < 4) {
log.info("Usage: \n param 1 - path to file \n param 2 - stream name to send the data \n param 3 - time in ms to wait to send each data \n param 4 - broker list");
} else {
Producer<String, String> producer = new Producer<String, String>(createProducerConfig(args[3]));
Gson gson = new Gson();
Reader in = new FileReader(args[0]);
CSVParser parser = CSVFormat.DEFAULT.parse(in);
List<String> columnNames = new ArrayList<>();
for (CSVRecord csvRecord : parser.getRecords()) {
if (columnNames.size() == 0) {
Iterator<String> iterator = csvRecord.iterator();
while (iterator.hasNext()) {
columnNames.add(iterator.next());
}
} else {
StratioStreamingMessage message = new StratioStreamingMessage();
message.setOperation(STREAM_OPERATIONS.MANIPULATION.INSERT.toLowerCase());
message.setStreamName(args[1]);
message.setTimestamp(System.currentTimeMillis());
message.setSession_id(String.valueOf(System.currentTimeMillis()));
message.setRequest_id(String.valueOf(System.currentTimeMillis()));
message.setRequest("dummy request");
List<ColumnNameTypeValue> sensorData = new ArrayList<>();
for (int i = 0; i < columnNames.size(); i++) {
// Workaround
Object value = null;
try {
value = Double.valueOf(csvRecord.get(i));
} catch (NumberFormatException e) {
value = csvRecord.get(i);
}
sensorData.add(new ColumnNameTypeValue(columnNames.get(i), null, value));
}
message.setColumns(sensorData);
String json = gson.toJson(message);
log.info("Sending data: {}", json);
producer.send(new KeyedMessage<String, String>(InternalTopic.TOPIC_DATA.getTopicName(), STREAM_OPERATIONS.MANIPULATION.INSERT, json));
log.info("Sleeping {} ms...", args[2]);
Thread.sleep(Long.valueOf(args[2]));
}
}
log.info("Program completed.");
}
}Example 14
| Project: fullstop-master File: CredentialReportCSVParserImpl.java View source code |
@Override
public List<CSVReportEntry> apply(final GetCredentialReportResult report) {
Assert.state(Textcsv.toString().equals(report.getReportFormat()), "unknown credential report format: " + report.getReportFormat());
try (final Reader r = new BufferedReader(new InputStreamReader(new ByteBufferBackedInputStream(report.getContent())))) {
final CSVParser parser = new CSVParser(r, CSV_FORMAT);
final Map<String, Integer> headers = parser.getHeaderMap();
Assert.state(headers.containsKey("user"), "Header 'user' not found in CSV");
Assert.state(headers.containsKey("arn"), "Header 'arn' not found in CSV");
Assert.state(headers.containsKey("password_enabled"), "Header 'password_enabled' not found in CSV");
Assert.state(headers.containsKey("mfa_active"), "Header 'mfa_active' not found in CSV");
Assert.state(headers.containsKey("access_key_1_active"), "Header 'access_key_1_active' not found in CSV");
Assert.state(headers.containsKey("access_key_2_active"), "Header 'access_key_2_active' not found in CSV");
return stream(parser.spliterator(), false).map(this::toCSVReportEntry).filter(Objects::nonNull).collect(toList());
} catch (final IOException e) {
throw new RuntimeException("Could not read csv report", e);
}
}Example 15
| Project: geode-social-demo-master File: Application.java View source code |
public void submitPosts() throws IOException, InterruptedException {
List<String> names = loadNames();
for (String name : names) {
people.save(new Person(name));
}
Random random = new Random();
Reader in = new FileReader(new File("data", "/Sentiment Analysis Dataset.csv"));
// Some of the data has comments. Don't use any comment characters or escape
// characters
CSVParser tweets = CSVFormat.newFormat(',').withHeader().parse(in);
int tweetCount = 0;
for (CSVRecord tweet : tweets) {
String name = names.get(random.nextInt(names.size()));
String post = tweet.get("SentimentText");
posts.save(new Post(name, post));
tweetCount++;
System.out.println(name + ": " + post);
Thread.sleep(1000);
if (System.in.available() > 0) {
System.out.println("Exiting...");
System.out.println("Posted " + tweetCount + " tweets");
break;
}
}
}Example 16
| Project: JAVMovieScraper-master File: WebReleaseRenamer.java View source code |
public List<CSVRecord> readFromCSVFile(String filePath) throws IOException {
CSVFormat format = CSVFormat.RFC4180.withDelimiter(',').withCommentMarker('#');
try (InputStream inputStream = getClass().getResourceAsStream(filePath);
CSVParser parser = new CSVParser(new InputStreamReader(inputStream), format)) {
List<CSVRecord> csvRecords = parser.getRecords();
return csvRecords;
}
}Example 17
| Project: marketcetera-master File: CSVFeed.java View source code |
/* (non-Javadoc)
* @see java.lang.Runnable#run()
*/
@Override
public void run() {
SLF4JLoggerProxy.debug(CSVFeed.class, //$NON-NLS-1$
"Beginning request {}", this);
isRunning.set(true);
try {
CSVParser parser = null;
long start = System.currentTimeMillis();
long count = 0;
long delay = credentials.getReplayRate();
if (delay < 0) {
delay = 0;
}
while (isRunning.get()) {
if (parser == null) {
start = System.currentTimeMillis();
count = 0;
parser = new CSVParser(new FileReader(new File(credentials.getMarketdataDirectory(), dataFilename)), CSVStrategy.EXCEL_STRATEGY);
}
String[] line = parser.getLine();
if (line == null) {
Messages.END_OF_DATA_REACHED.debug(org.marketcetera.core.Messages.USER_MSG_CATEGORY, count, System.currentTimeMillis() - start);
if (credentials.getReplayEvents() && isRunning.get()) {
count = 0;
parser = null;
continue;
} else {
break;
}
}
if (delay != 0) {
Thread.sleep(delay);
}
count += 1;
dataReceived(handle, CSVQuantum.getQuantum(line, request, credentials.getReplayRate()));
}
} catch (Exception e) {
REQUEST_FAILED.warn(org.marketcetera.core.Messages.USER_MSG_CATEGORY, e, this);
} finally {
isRunning.set(false);
}
}Example 18
| Project: nifi-master File: CSVHeaderSchemaStrategy.java View source code |
@Override
public RecordSchema getSchema(final FlowFile flowFile, final InputStream contentStream) throws SchemaNotFoundException {
if (this.context == null) {
throw new SchemaNotFoundException("Schema Access Strategy intended only for validation purposes and cannot obtain schema");
}
try {
final CSVFormat csvFormat = CSVUtils.createCSVFormat(context).withFirstRecordAsHeader();
try (final Reader reader = new InputStreamReader(new BOMInputStream(contentStream));
final CSVParser csvParser = new CSVParser(reader, csvFormat)) {
final List<RecordField> fields = new ArrayList<>();
for (final String columnName : csvParser.getHeaderMap().keySet()) {
fields.add(new RecordField(columnName, RecordFieldType.STRING.getDataType()));
}
return new SimpleRecordSchema(fields);
}
} catch (final Exception e) {
throw new SchemaNotFoundException("Failed to read Header line from CSV", e);
}
}Example 19
| Project: nz.co.kakariki.WNMSExtract-master File: BorgBlockReader.java View source code |
@Override
public void readAll() {
ArrayList<ColumnStructure> colstruct = new ArrayList<ColumnStructure>();
colstruct.add(ColumnStructure.VC);
colstruct.add(ColumnStructure.TS);
colstruct.add(ColumnStructure.FL);
for (RncAp r : EnumSet.allOf(RncAp.class)) {
for (AppProc a : EnumSet.allOf(AppProc.class)) {
ArrayList<ArrayList<String>> mapmap = new ArrayList<ArrayList<String>>();
try {
URL borg = new URL(BORG + r.getFile(a));
URLConnection conn = borg.openConnection();
BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
CSVParser parser = new CSVParser(in, strategy);
//if header
String[] header = parser.getLine();
//and body
String[] line = null;
while ((line = parser.getLine()) != null) {
Calendar cal = Calendar.getInstance();
cal.setTime(BORG_DF.parse(line[0]));
String datestr = ALUDBUtilities.ALUDB_DF.format(cal.getTime());
//System.out.println(r.toString()+"/"+a.toString()+"//"+line[0]+"///"+datestr);
for (int i = 2; i < line.length; i++) {
ArrayList<String> map = new ArrayList<String>();
map.add(0, idConvert(r.toString(), a.toString(), header[i]));
map.add(1, datestr);
map.add(2, line[i]);
mapmap.add(map);
}
}
in.close();
} catch (MalformedURLException mrue) {
System.err.println("Borg Path incorrect " + mrue);
} catch (IOException ioe) {
System.err.println("Cannot read Borg file " + ioe);
} catch (ParseException pe) {
System.err.println("Cannot parse Date field " + pe);
}
ALUDBUtilities.insert(databasetype, TABLE, colstruct, mapmap);
}
}
}Example 20
| Project: openiam-idm-master File: CSVAdapter.java View source code |
public SyncResponse startSynch(SynchConfig config) {
log.debug("CSV startSynch CALLED.^^^^^^^^");
System.out.println("CSV startSynch CALLED.^^^^^^^^");
Reader reader = null;
MatchObjectRule matchRule = null;
provService = (ProvisionService) ac.getBean("defaultProvision");
String requestId = UUIDGen.getUUID();
IdmAuditLog synchStartLog = new IdmAuditLog();
synchStartLog.setSynchAttributes("SYNCH_USER", config.getSynchConfigId(), "START", "SYSTEM", requestId);
synchStartLog = auditHelper.logEvent(synchStartLog);
try {
matchRule = matchRuleFactory.create(config);
} catch (ClassNotFoundException cnfe) {
log.error(cnfe);
cnfe.printStackTrace();
synchStartLog.updateSynchAttributes("FAIL", ResponseCode.CLASS_NOT_FOUND.toString(), cnfe.toString());
auditHelper.logEvent(synchStartLog);
SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
resp.setErrorCode(ResponseCode.CLASS_NOT_FOUND);
return resp;
}
File file = new File(config.getFileName());
try {
reader = new FileReader(file);
} catch (FileNotFoundException fe) {
fe.printStackTrace();
log.error(fe);
synchStartLog.updateSynchAttributes("FAIL", ResponseCode.FILE_EXCEPTION.toString(), fe.toString());
auditHelper.logEvent(synchStartLog);
SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
resp.setErrorCode(ResponseCode.FILE_EXCEPTION);
return resp;
}
CSVParser parser = new CSVParser(reader, CSVStrategy.EXCEL_STRATEGY);
try {
int ctr = 0;
String[][] fileContentAry = parser.getAllValues();
int size = fileContentAry.length;
for (String[] lineAry : fileContentAry) {
log.debug("File Row #= " + lineAry[0]);
System.out.println("File Row #= " + lineAry[0]);
if (ctr == 0) {
populateTemplate(lineAry);
ctr++;
} else {
//populate the data object
pUser = new ProvisionUser();
LineObject rowObj = rowHeader.copy();
populateRowObject(rowObj, lineAry);
try {
System.out.println("Validation being called");
// validate
if (config.getValidationRule() != null && config.getValidationRule().length() > 0) {
ValidationScript script = SynchScriptFactory.createValidationScript(config.getValidationRule());
int retval = script.isValid(rowObj);
if (retval == ValidationScript.NOT_VALID) {
log.debug("Validation failed...");
// log this object in the exception log
}
if (retval == ValidationScript.SKIP) {
continue;
}
}
System.out.println("Getting column map...");
// check if the user exists or not
Map<String, Attribute> rowAttr = rowObj.getColumnMap();
System.out.println("Row Attr..." + rowAttr);
//
matchRule = matchRuleFactory.create(config);
User usr = matchRule.lookup(config, rowAttr);
System.out.println("Preparing transform script");
// transform
if (config.getTransformationRule() != null && config.getTransformationRule().length() > 0) {
TransformScript transformScript = SynchScriptFactory.createTransformationScript(config.getTransformationRule());
// initialize the transform script
if (usr != null) {
transformScript.setNewUser(false);
transformScript.setUser(userMgr.getUserWithDependent(usr.getUserId(), true));
transformScript.setPrincipalList(loginManager.getLoginByUser(usr.getUserId()));
transformScript.setUserRoleList(roleDataService.getUserRolesAsFlatList(usr.getUserId()));
} else {
transformScript.setNewUser(true);
}
System.out.println("Execute transform script");
int retval = transformScript.execute(rowObj, pUser);
System.out.println("Execute complete transform script");
if (retval == TransformScript.DELETE && pUser.getUser() != null) {
provService.deleteByUserId(pUser, UserStatusEnum.DELETED, systemAccount);
} else {
// call synch
if (retval != TransformScript.DELETE) {
if (usr != null) {
System.out.println("Updating existing user");
pUser.setUserId(usr.getUserId());
provService.modifyUser(pUser);
} else {
System.out.println("New user being provisioned");
pUser.setUserId(null);
provService.addUser(pUser);
}
}
}
}
// show the user object
} catch (ClassNotFoundException cnfe) {
log.error(cnfe);
synchStartLog.updateSynchAttributes("FAIL", ResponseCode.CLASS_NOT_FOUND.toString(), cnfe.toString());
auditHelper.logEvent(synchStartLog);
SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
resp.setErrorCode(ResponseCode.CLASS_NOT_FOUND);
return resp;
}
}
}
} catch (IOException io) {
io.printStackTrace();
synchStartLog.updateSynchAttributes("FAIL", ResponseCode.IO_EXCEPTION.toString(), io.toString());
auditHelper.logEvent(synchStartLog);
SyncResponse resp = new SyncResponse(ResponseStatus.FAILURE);
resp.setErrorCode(ResponseCode.IO_EXCEPTION);
return resp;
}
log.debug("CSV SYNCHRONIZATION COMPLETE^^^^^^^^");
System.out.println("CSV SYNCHRONIZATION COMPLETE^^^^^^^^");
SyncResponse resp = new SyncResponse(ResponseStatus.SUCCESS);
return resp;
}Example 21
| Project: Phenex-master File: PhenotypeProposalsLoader.java View source code |
public void loadProposals(File file) throws IOException {
final CSVParser parser = new CSVParser(new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8")));
final String[][] csv = parser.getAllValues();
boolean first = true;
for (String[] line : csv) {
if (first) {
this.header = Arrays.asList(line);
first = false;
} else {
if (!StringUtils.isBlank(get("stateid", line))) {
this.addProposalToDataSet(this.createProposal(line));
}
}
}
}Example 22
| Project: structr-master File: FromCsvFunction.java View source code |
@Override
public Object apply(final ActionContext ctx, final Object caller, final Object[] sources) {
if (arrayHasMinLengthAndMaxLengthAndAllElementsNotNull(sources, 1, 4)) {
try {
final List<Map<String, String>> objects = new LinkedList<>();
final String source = sources[0].toString();
String delimiter = ";";
String quoteChar = "\"";
String recordSeparator = "\n";
switch(sources.length) {
case 4:
recordSeparator = (String) sources[3];
case 3:
quoteChar = (String) sources[2];
case 2:
delimiter = (String) sources[1];
break;
}
CSVFormat format = CSVFormat.newFormat(delimiter.charAt(0)).withHeader();
format = format.withQuote(quoteChar.charAt(0));
format = format.withRecordSeparator(recordSeparator);
format = format.withIgnoreEmptyLines(true);
format = format.withIgnoreSurroundingSpaces(true);
format = format.withSkipHeaderRecord(true);
format = format.withQuoteMode(QuoteMode.ALL);
CSVParser parser = new CSVParser(new StringReader(source), format);
for (final CSVRecord record : parser.getRecords()) {
objects.add(record.toMap());
}
return objects;
} catch (Throwable t) {
logException(t, "{}: Exception for parameter: {}", new Object[] { getName(), getParametersAsString(sources) });
}
return "";
} else {
logParameterError(caller, sources, ctx.isJavaScriptContext());
}
return usage(ctx.isJavaScriptContext());
}Example 23
| Project: camel-master File: CsvUnmarshaller.java View source code |
@Override
public Object unmarshal(Exchange exchange, InputStream inputStream) throws IOException {
Reader reader = null;
try {
reader = new InputStreamReader(inputStream, IOHelper.getCharsetName(exchange));
CSVParser parser = new CSVParser(reader, format);
CsvIterator answer = new CsvIterator(parser, converter);
// add to UoW so we can close the iterator so it can release any resources
exchange.addOnCompletion(new CsvUnmarshalOnCompletion(answer));
return answer;
} catch (Exception e) {
IOHelper.close(reader);
throw e;
}
}Example 24
| Project: csv2xml-master File: CcsCsvReader.java View source code |
void load() throws IOException {
File file = new File(fileLocation);
file = removeEmptyLinesFromCsv(file);
if (containsCopyright) {
copyright = removeTrailingCharacters(tail(file), ',');
file = removeLastLine(file);
}
InputStreamReader isReader = new InputStreamReader(new FileInputStream(file), "UTF-8");
csvParser = new CSVParser(isReader, CSVStrategy.EXCEL_STRATEGY);
firstLine = csvParser.getLine();
getNextRecord();
}Example 25
| Project: gephi-master File: SpreadsheetUtils.java View source code |
public static CSVParser configureCSVParser(File file, Character fieldSeparator, Charset charset, boolean withFirstRecordAsHeader) throws IOException { if (fieldSeparator == null) { fieldSeparator = ','; } CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(fieldSeparator).withIgnoreEmptyLines(true).withNullString("").withIgnoreSurroundingSpaces(true).withTrim(true); if (withFirstRecordAsHeader) { csvFormat = csvFormat.withFirstRecordAsHeader().withAllowMissingColumnNames(false).withIgnoreHeaderCase(false); } else { csvFormat = csvFormat.withHeader((String[]) null).withSkipHeaderRecord(false); } FileInputStream fileInputStream = new FileInputStream(file); InputStreamReader is = new InputStreamReader(fileInputStream, charset); return new CSVParser(is, csvFormat); }
Example 26
| Project: hibiscus.depotviewer-master File: CSVImportConfigDialog.java View source code |
private void reload() throws RemoteException {
this.getError().setValue("");
boolean enable = true;
list.clear();
header.clear();
Integer headerline = ((Integer) getSkipLines().getValue());
try {
SWTUtil.disposeChildren(this.comp);
comp.setLayoutData(new GridData(GridData.FILL_BOTH));
this.comp.setLayout(new GridLayout());
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), Charset.forName((String) getCharset().getValue())));
String line = null;
int counter = 0;
while ((line = br.readLine()) != null && counter < 30) {
GenericObjectHashMap g = new GenericObjectHashMap();
g.setAttribute("Dateiinhalt", line);
list.add(g);
counter++;
}
header.add("Dateiinhalt");
br.close();
FileInputStream is = new FileInputStream(file);
InputStreamReader isr = new InputStreamReader(is, Charset.forName((String) getCharset().getValue()));
CSVFormat format = CSVFormat.RFC4180.withDelimiter(((String) getTrennzeichen().getValue()).charAt(0)).withSkipLines(headerline - 1);
CSVParser parser = new CSVParser(isr, format);
list.clear();
boolean iserror = false;
boolean isheader = true;
header.clear();
for (CSVRecord record : parser) {
if (isheader) {
for (int i = 0; i < record.size(); i++) {
String name = record.get(i);
if (name.isEmpty()) {
name = "Namenlos";
}
String neuername = name;
counter = 1;
while (header.contains(neuername)) {
neuername = name + " (" + counter + ")";
counter++;
}
header.add(neuername);
}
isheader = false;
continue;
}
GenericObjectHashMap g = new GenericObjectHashMap();
g.setAttribute("_DEPOTVIEWER_IGNORE", "");
g.setAttribute("_DEPOTVIEWER_IDX", "" + (record.getRecordNumber() + headerline));
for (int i = 0; i < header.size(); i++) {
if (i >= record.size()) {
g.setAttribute("_DEPOTVIEWER_IGNORE", "X");
iserror = true;
continue;
}
g.setAttribute(header.get(i), record.get(i));
}
list.add(g);
}
if (iserror) {
getError().setValue("Die mit X markierten Zeilen werden ignoriert.");
}
parser.close();
} catch (Exception e) {
Logger.error("unable to read file", e);
this.getError().setValue("Fehler beim Lesen der Datei:\n" + e.getMessage());
enable = false;
}
TablePart tab = new TablePart(list, null);
tab.addColumn("" + headerline, "_DEPOTVIEWER_IDX");
tab.addColumn("", "_DEPOTVIEWER_IGNORE");
for (String h : header) {
tab.addColumn(h, h);
}
tab.paint(comp);
comp.layout(true);
if (weiterbutton != null) {
weiterbutton.setEnabled(enable);
}
}Example 27
| Project: jagger-master File: CsvProvider.java View source code |
private void init() {
if (path == null) {
throw new TechnicalException("File path can't be NULL!");
}
try {
parser = new CSVParser(new BufferedReader(new FileReader(new File(path))), strategy);
} catch (FileNotFoundException e) {
throw Throwables.propagate(e);
}
if (readHeader) {
try {
objectCreator.setHeader(parser.getLine());
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
}Example 28
| Project: mathosphere-master File: ConverterPairCSVToMatrix.java View source code |
public static void main(String[] args) throws Exception {
System.out.println("number of args given = " + args.length);
String in = "/home/felix/170113run";
if (args.length == 0) {
System.out.println("input file name? ");
BufferedReader buffer = new BufferedReader(new InputStreamReader(System.in));
in = buffer.readLine().trim();
} else {
in = args[0];
}
final String outbase = in + "_out_";
List<String> rows = null;
List<String> cols = null;
for (int i = 0; i < 5; i++) {
System.out.println("parsing file to CSV: " + in);
final CSVParser parser = CSVParser.parse(new File(in), Charset.defaultCharset(), CSV_FORMAT);
System.out.println("finished parsing");
System.out.println("creating matrix (" + i + ")");
final HashMap<Tuple2<String, String>, Double> matrix = new HashMap<>();
for (CSVRecord row : parser) {
Tuple2<String, String> key = getDocumentIDsFromRow(row);
if (matrix.containsKey(key)) {
throw new RuntimeException("matrix already contains key: " + key);
}
matrix.put(key, getDistanceFromRow(row, i));
}
System.out.println("finished creating matrix (" + i + ")");
if (rows == null && cols == null) {
final List<String>[] rowsAndCols = mergeKeysIntoRowsAndCols(matrix);
rows = rowsAndCols[0];
cols = rowsAndCols[1];
} else {
System.out.println("reusing previously created rows and columns");
}
System.out.println("writing matrix");
writeOrderedMatrix(matrix, outbase + i + ".csv", rows, cols);
System.out.println("finished writing matrix");
// reset parser
parser.close();
}
}Example 29
| Project: migration-tools-master File: CsvInput.java View source code |
@Override
protected void init(Reader reader) {
CsvFormatBuilder builder = new CsvFormatBuilder(this);
CSVFormat format = builder.build();
Character quote = builder.getQuote();
doubleQuote = valueOf(quote) + valueOf(quote);
try {
parser = new CSVParser(reader, format);
iterator = parser.iterator();
} catch (IOException exception) {
throw new InputException(exception);
}
}Example 30
| Project: nuxeo-master File: UserProfileImporter.java View source code |
public void doImport(CoreSession session) {
UserProfileService ups = Framework.getLocalService(UserProfileService.class);
config = ups.getImporterConfig();
if (config == null) {
log.error("No importer configuration could be found");
return;
}
dataFileName = config.getDataFileName();
if (dataFileName == null) {
log.error("No importer dataFileName was supplied");
return;
}
InputStream is = getResourceAsStream(dataFileName);
if (is == null) {
log.error("Error locating CSV data file: " + dataFileName);
return;
}
Reader in = new BufferedReader(new InputStreamReader(is));
CSVParser parser = null;
try {
parser = CSVFormat.DEFAULT.withEscape(escapeCharacter).withHeader().parse(in);
doImport(session, parser, ups);
} catch (IOException e) {
log.error("Unable to read CSV file", e);
} finally {
if (parser != null) {
try {
parser.close();
} catch (IOException e) {
log.debug(e, e);
}
}
}
}Example 31
| Project: Viz-master File: SpreadsheetUtils.java View source code |
public static CSVParser configureCSVParser(File file, Character fieldSeparator, Charset charset, boolean withFirstRecordAsHeader) throws IOException { if (fieldSeparator == null) { fieldSeparator = ','; } CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(fieldSeparator).withIgnoreEmptyLines(true).withNullString("").withIgnoreSurroundingSpaces(true).withTrim(true); if (withFirstRecordAsHeader) { csvFormat = csvFormat.withFirstRecordAsHeader().withAllowMissingColumnNames(false).withIgnoreHeaderCase(false); } else { csvFormat = csvFormat.withHeader((String[]) null).withSkipHeaderRecord(false); } FileInputStream fileInputStream = new FileInputStream(file); InputStreamReader is = new InputStreamReader(fileInputStream, charset); return new CSVParser(is, csvFormat); }
Example 32
| Project: ambari-master File: CsvSerializerTest.java View source code |
@Test
public void testSerializeResources_NoColumnInfo() throws Exception {
Result result = new ResultImpl(true);
result.setResultStatus(new ResultStatus(ResultStatus.STATUS.OK));
TreeNode<Resource> tree = result.getResultTree();
List<TreeMap<String, Object>> data = new ArrayList<TreeMap<String, Object>>() {
{
add(new TreeMap<String, Object>() {
{
put("property1", "value1a");
put("property2", "value2a");
put("property3", "value3a");
put("property4", "value4a");
}
});
add(new TreeMap<String, Object>() {
{
put("property1", "value1'b");
put("property2", "value2'b");
put("property3", "value3'b");
put("property4", "value4'b");
}
});
add(new TreeMap<String, Object>() {
{
put("property1", "value1,c");
put("property2", "value2,c");
put("property3", "value3,c");
put("property4", "value4,c");
}
});
}
};
tree.setName("items");
tree.setProperty("isCollection", "true");
addChildResource(tree, "resource", 0, data.get(0));
addChildResource(tree, "resource", 1, data.get(1));
addChildResource(tree, "resource", 2, data.get(2));
replayAll();
//execute test
Object o = new CsvSerializer().serialize(result).toString().replace("\r", "");
verifyAll();
assertNotNull(o);
StringReader reader = new StringReader(o.toString());
CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT);
List<CSVRecord> records = csvParser.getRecords();
assertNotNull(records);
assertEquals(3, records.size());
int i = 0;
for (CSVRecord record : records) {
TreeMap<String, Object> actualData = data.get(i++);
assertEquals(actualData.size(), record.size());
for (String item : record) {
assertTrue(actualData.containsValue(item));
}
}
csvParser.close();
}Example 33
| Project: autopsy-master File: CreditCards.java View source code |
/**
* Load the BIN range information from disk. If the map has already been
* initialized, don't load again.
*/
private static synchronized void loadBINRanges() {
if (binsLoaded == false) {
try {
//NON-NLS
InputStreamReader in = new InputStreamReader(CreditCards.class.getResourceAsStream("ranges.csv"));
CSVParser rangesParser = CSVFormat.RFC4180.withFirstRecordAsHeader().parse(in);
//parse each row and add to range map
for (CSVRecord record : rangesParser) {
/**
* Because ranges.csv allows both 6 and (the newer) 8 digit
* BINs, but we need a consistent length for the range map,
* we pad all the numbers out to 8 digits
*/
//pad start with 0's //NON-NLS
String start = StringUtils.rightPad(record.get("iin_start"), 8, "0");
//if there is no end listed, use start, since ranges will be closed.
//NON-NLS
String end = StringUtils.defaultIfBlank(record.get("iin_end"), start);
//pad end with 9's //NON-NLS
end = StringUtils.rightPad(end, 8, "99");
//NON-NLS
final String numberLength = record.get("number_length");
try {
BINRange binRange = new BINRange(Integer.parseInt(start), Integer.parseInt(end), StringUtils.isBlank(numberLength) ? null : Integer.valueOf(numberLength), //NON-NLS
record.get("scheme"), //NON-NLS
record.get("brand"), //NON-NLS
record.get("type"), //NON-NLS
record.get("country"), //NON-NLS
record.get("bank_name"), //NON-NLS
record.get("bank_url"), //NON-NLS
record.get("bank_phone"), //NON-NLS
record.get("bank_city"));
binRanges.put(Range.closed(binRange.getBINstart(), binRange.getBINend()), binRange);
} catch (NumberFormatException numberFormatException) {
LOGGER.log(Level.WARNING, "Failed to parse BIN range: " + record.toString(), numberFormatException);
}
binsLoaded = true;
}
} catch (IOException ex) {
LOGGER.log(Level.WARNING, "Failed to load BIN ranges form ranges.csv", ex);
MessageNotifyUtil.Notify.warn("Credit Card Number Discovery", "There was an error loading Bank Identification Number information. Accounts will not have their BINs identified.");
}
}
}Example 34
| Project: aws-big-data-blog-master File: LambdaContainer.java View source code |
//Validation/Conversion Layer function
public void validateAndNormalizeInputData(S3Event event, Context ctx) throws Exception {
AmazonS3 s3Client;
InputStream inputFileStream = null;
InputStream readableDataStream = null;
List<S3EventNotificationRecord> notificationRecords = event.getRecords();
s3Client = new AmazonS3Client();
String eventFileName, siteName, dbfName;
CSVParser fileParser = null;
for (S3EventNotificationRecord record : notificationRecords) {
eventFileName = record.getS3().getObject().getKey();
S3Object s3Object = s3Client.getObject(new GetObjectRequest(record.getS3().getBucket().getName(), record.getS3().getObject().getKey()));
inputFileStream = s3Object.getObjectContent();
fileParser = new CSVParser(new InputStreamReader(inputFileStream), CSVFormat.TDF.withCommentMarker('-'));
List<CSVRecord> records = fileParser.getRecords();
StringWriter writer = new StringWriter();
CSVPrinter printer = null;
if (records.get(0).toString().matches(".*[^0-9].*")) {
records.remove(0);
}
printer = new CSVPrinter(writer, CSVFormat.DEFAULT.withRecordSeparator(System.getProperty("line.separator")));
printer.printRecords(records);
printer.flush();
readableDataStream = new ByteArrayInputStream(writer.toString().getBytes("utf-8"));
s3Client.putObject(record.getS3().getBucket().getName(), "validated/" + eventFileName + ".csv", readableDataStream, new ObjectMetadata());
printer.close();
readableDataStream.close();
}
}Example 35
| Project: kylo-master File: CSVFileSchemaParser.java View source code |
@Override
public Schema parse(InputStream is, Charset charset, TableSchemaType target) throws IOException {
Validate.notNull(target, "target must not be null");
Validate.notNull(is, "stream must not be null");
Validate.notNull(charset, "charset must not be null");
validate();
// Parse the file
String sampleData = ParserHelper.extractSampleLines(is, charset, numRowsToSample);
Validate.notEmpty(sampleData, "No data in file");
CSVFormat format = createCSVFormat(sampleData);
try (Reader reader = new StringReader(sampleData)) {
CSVParser parser = format.parse(reader);
DefaultFileSchema fileSchema = populateSchema(parser);
fileSchema.setCharset(charset.name());
// Convert to target schema with proper derived types
Schema targetSchema = convertToTarget(target, fileSchema);
return targetSchema;
}
}Example 36
| Project: MarsImagesAndroid-master File: Rover.java View source code |
public List<CSVRecord> siteLocationData(int siteIndex) {
//return cached results if we have them
//TODO invalidate cache when needed (after a day?)
List<CSVRecord> cachedLocations = locationsBySite.get(siteIndex);
if (cachedLocations != null)
return cachedLocations;
List<CSVRecord> locations = new ArrayList<>();
URL locationsURL = null;
try {
String sixDigitSite = String.format("%06d", siteIndex);
locationsURL = new URL(getURLPrefix() + "/locations/site_" + sixDigitSite + ".csv");
Log.d(TAG, "location url: %@" + locationsURL);
HttpURLConnection urlConnection = (HttpURLConnection) locationsURL.openConnection();
try {
InputStream in = new BufferedInputStream(urlConnection.getInputStream());
final Reader reader = new InputStreamReader(in);
final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT);
try {
for (final CSVRecord record : parser) {
locations.add(record);
}
locationsBySite.put(siteIndex, locations);
} finally {
parser.close();
reader.close();
}
} finally {
urlConnection.disconnect();
}
} catch (MalformedURLException e) {
Log.e(TAG, "Badly formatted URL for location manifest: " + locationsURL);
} catch (IOException e) {
Log.e(TAG, "Error reading from location manifest URL: " + locationsURL);
}
return locations;
}Example 37
| Project: myria-master File: FileScan.java View source code |
@Override
protected void init(final ImmutableMap<String, Object> execEnvVars) throws DbException {
buffer = new TupleBatchBuffer(getSchema());
try {
parser = new CSVParser(new BufferedReader(new InputStreamReader(source.getInputStream())), CSVFormat.newFormat(delimiter).withQuote(quote).withEscape(escape));
iterator = parser.iterator();
for (int i = 0; i < numberOfSkippedLines; i++) {
iterator.next();
}
} catch (IOException e) {
throw new DbException(e);
}
lineNumber = 0;
}Example 38
| Project: nuxeo-services-master File: SQLHelper.java View source code |
private void loadData() throws DirectoryException {
log.debug("loading data file: " + dataFileName);
CSVParser csvParser = null;
PreparedStatement ps = null;
try {
InputStream is = getClass().getClassLoader().getResourceAsStream(dataFileName);
if (is == null) {
is = Framework.getResourceLoader().getResourceAsStream(dataFileName);
if (is == null) {
throw new DirectoryException("data file not found: " + dataFileName);
}
}
csvParser = new CSVParser(new InputStreamReader(is, SQL_SCRIPT_CHARSET), CSVFormat.DEFAULT.withDelimiter(characterSeparator).withHeader());
Map<String, Integer> header = csvParser.getHeaderMap();
List<Column> columns = new ArrayList<>();
Insert insert = new Insert(table);
for (String columnName : header.keySet()) {
String trimmedColumnName = columnName.trim();
Column column = table.getColumn(trimmedColumnName);
if (column == null) {
throw new DirectoryException("column not found: " + trimmedColumnName);
}
columns.add(table.getColumn(trimmedColumnName));
insert.addColumn(column);
}
String insertSql = insert.getStatement();
log.debug("insert statement: " + insertSql);
ps = connection.prepareStatement(insertSql);
for (CSVRecord record : csvParser) {
if (record.size() == 0 || record.size() == 1 && StringUtils.isBlank(record.get(0))) {
// empty lines
continue;
}
if (!record.isConsistent()) {
log.error("invalid column count while reading CSV file: " + dataFileName + ", values: " + record);
continue;
}
if (logger.isLogEnabled()) {
List<Serializable> values = new ArrayList<>(header.size());
for (String value : record) {
if (SQL_NULL_MARKER.equals(value)) {
value = null;
}
values.add(value);
}
logger.logSQL(insertSql, values);
}
for (int i = 0; i < header.size(); i++) {
Column column = columns.get(i);
String value = record.get(i);
Serializable v;
try {
if (SQL_NULL_MARKER.equals(value)) {
v = null;
} else if (column.getType().spec == ColumnSpec.STRING) {
v = value;
} else if (column.getType().spec == ColumnSpec.BOOLEAN) {
v = Boolean.valueOf(value);
} else if (column.getType().spec == ColumnSpec.LONG) {
v = Long.valueOf(value);
} else if (column.getType().spec == ColumnSpec.TIMESTAMP) {
Calendar cal = new GregorianCalendar();
cal.setTime(Timestamp.valueOf(value));
v = cal;
} else if (column.getType().spec == ColumnSpec.DOUBLE) {
v = Double.valueOf(value);
} else {
throw new DirectoryException("unrecognized column type: " + column.getType() + ", values: " + record);
}
column.setToPreparedStatement(ps, i + 1, v);
} catch (IllegalArgumentException e) {
throw new DirectoryException(String.format("failed to set column '%s' on table '%s', values: %s", column.getPhysicalName(), table.getPhysicalName(), record), e);
} catch (SQLException e) {
throw new DirectoryException(String.format("Table '%s' initialization failed: %s, values: %s", table.getPhysicalName(), e.getMessage(), record), e);
}
}
ps.execute();
}
} catch (IOException e) {
throw new DirectoryException("Read error while reading data file: " + dataFileName, e);
} catch (SQLException e) {
throw new DirectoryException(String.format("Table '%s' initialization failed: %s", table.getPhysicalName(), e.getMessage()), e);
} finally {
DirectoryException e = new DirectoryException();
try {
if (csvParser != null) {
csvParser.close();
}
} catch (IOException ioe) {
e.addSuppressed(ioe);
}
try {
if (ps != null) {
ps.close();
}
} catch (SQLException sqle) {
e.addSuppressed(sqle);
}
if (e.getSuppressed().length > 0) {
throw e;
}
}
}Example 39
| Project: pippo-master File: CsvEngine.java View source code |
@Override
@SuppressWarnings("unchecked")
public <T> T fromString(String content, Class<T> classOfT) {
if (!classOfT.isArray()) {
if (Collection.class.isAssignableFrom(classOfT)) {
// Collections are NOT supported for deserialization from CSV
throw new RuntimeException("Collection types are not supported. Please specify an array[] type.");
}
throw new RuntimeException(String.format("Array[] types are required. Please specify %s[]", classOfT.getName()));
}
Class<?> objectType = classOfT.getComponentType();
int currentLine = 0;
try (CSVParser parser = new CSVParser(new StringReader(content), getCSVFormat().withHeader())) {
Set<String> columns = parser.getHeaderMap().keySet();
Map<String, Field> fieldMap = getFieldMap(objectType);
Constructor<?> objectConstructor;
try {
objectConstructor = objectType.getConstructor();
} catch (NoSuchMethodException e) {
throw new RuntimeException("A default constructor is required for " + objectType.getName());
}
List objects = new ArrayList<>();
for (CSVRecord record : parser) {
currentLine++;
Object o = objectConstructor.newInstance();
for (String column : columns) {
Field field = fieldMap.get(caseSensitiveFieldNames ? column : column.toLowerCase());
String value = record.get(column);
Object object = objectFromString(value, field.getType());
field.set(o, object);
}
objects.add(o);
}
Object array = Array.newInstance(objectType, objects.size());
for (int i = 0; i < objects.size(); i++) {
Array.set(array, i, objects.get(i));
}
return (T) array;
} catch (Exception e) {
throw new RuntimeException("Failed to parse CSV near line #" + currentLine, e);
}
}Example 40
| Project: qalingo-i18n-master File: LoaderTranslationUtil.java View source code |
public static final void buildMessagesProperties(String currentPath, String project, String filePath, List<String> activedLanguages, String defaultLanguage, String inputEncoding, String outputEncoding) {
try {
String newPath = currentPath + project + LoaderTranslation.PROPERTIES_PATH;
File folderProject = new File(newPath);
if (!folderProject.exists()) {
folderProject.mkdirs();
}
InputStreamReader reader = new InputStreamReader(new FileInputStream(new File(filePath)), inputEncoding);
LOG.info("File CSV encoding: " + inputEncoding);
LOG.info("File properties encoding: " + outputEncoding);
CSVFormat csvFormat = CSVFormat.DEFAULT;
CSVParser readerCSV = new CSVParser(reader, csvFormat);
List<CSVRecord> records = null;
try {
records = readerCSV.getRecords();
} catch (Exception e) {
LOG.error("Failed to load: " + filePath, e);
}
String prefixFileName = "";
CSVRecord firstLineRecord = records.get(0);
String[] prefixFileNameTemp = firstLineRecord.get(0).split("## Filename :");
if (prefixFileNameTemp.length > 1) {
prefixFileName = prefixFileNameTemp[1].trim();
}
String prefixKey = "";
CSVRecord secondLineRecord = records.get(1);
String[] prefixKeyTemp = secondLineRecord.get(0).split("## PrefixKey :");
if (prefixKeyTemp.length > 1) {
prefixKey = prefixKeyTemp[1].trim();
}
Map<String, Integer> availableLanguages = new HashMap<String, Integer>();
for (CSVRecord record : records) {
String firstCell = record.get(0);
if (firstCell.contains("Prefix") && record.size() > 1) {
String secondCell = record.get(1);
if (secondCell.contains("Key")) {
for (int i = 2; i < record.size(); i++) {
String languageCode = record.get(i);
availableLanguages.put(languageCode, new Integer(i));
}
break;
}
}
}
// BUILD DEFAULT FILE
String fileFullPath = newPath + "/" + prefixFileName + ".properties";
Integer defaultLanguagePosition = availableLanguages.get(defaultLanguage);
buildMessagesProperties(records, fileFullPath, prefixKey, defaultLanguage, defaultLanguagePosition, outputEncoding, true);
for (Iterator<String> iterator = availableLanguages.keySet().iterator(); iterator.hasNext(); ) {
String languageCode = (String) iterator.next();
if (activedLanguages.contains(languageCode)) {
Integer languagePosition = availableLanguages.get(languageCode);
String languegFileFullPath = newPath + "/" + prefixFileName + "_" + languageCode + ".properties";
buildMessagesProperties(records, languegFileFullPath, prefixKey, languageCode, languagePosition.intValue(), outputEncoding, false);
}
}
LOG.info(newPath + "/" + prefixFileName + ".properties");
} catch (Exception e) {
LOG.info("Exception", e);
}
}Example 41
| Project: ranger-master File: FileSourceUserGroupBuilder.java View source code |
public Map<String, List<String>> readTextFile(File textFile) throws Exception {
Map<String, List<String>> ret = new HashMap<String, List<String>>();
String delimiter = config.getUserSyncFileSourceDelimiter();
CSVFormat csvFormat = CSVFormat.newFormat(delimiter.charAt(0));
CSVParser csvParser = new CSVParser(new BufferedReader(new FileReader(textFile)), csvFormat);
List<CSVRecord> csvRecordList = csvParser.getRecords();
if (csvRecordList != null) {
for (CSVRecord csvRecord : csvRecordList) {
List<String> groups = new ArrayList<String>();
String user = csvRecord.get(0);
user = user.replaceAll("^\"|\"$", "");
int i = csvRecord.size();
for (int j = 1; j < i; j++) {
String group = csvRecord.get(j);
if (group != null && !group.isEmpty()) {
group = group.replaceAll("^\"|\"$", "");
groups.add(group);
}
}
ret.put(user, groups);
}
}
csvParser.close();
return ret;
}Example 42
| Project: SOCIETIES-Platform-master File: JungTest.java View source code |
public UndirectedSparseGraph makeGraph() {
InputStream is = null;
ArrayList<String[]> data = new ArrayList<String[]>();
File directory = new File(".");
try {
System.out.println("Current directory's canonical path: " + directory.getCanonicalPath());
System.out.println("Current directory's absolute path: " + directory.getAbsolutePath());
} catch (Exception e) {
System.out.println("Exceptione is =" + e.getMessage());
}
try {
CSVParser parser = new CSVParser(new FileReader("./src/test/resources/msn-data-xml.csv"), CSVStrategy.EXCEL_STRATEGY);
String[] value = parser.getLine();
while (value != null) {
data.add(value);
value = parser.getLine();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
int maxMem = 20000;
CISSimulator sim = new CISSimulator(10, 10);
final ArrayList<IActivity> actDiff = new ArrayList<IActivity>();
ApplicationContextLoader loader = new ApplicationContextLoader();
loader.load(sim, "SimTest-context.xml");
sim.getActFeed().setSessionFactory(sim.getSessionFactory());
cises = new ArrayList<ICisOwned>();
sim.setMaxActs(70000);
//cises.add(sim.simulate(1));
cises.add(sim.simulate("./src/test/resources/msn-data-xml.csv"));
final CPACreationPatterns cpa = new CPACreationPatterns();
class GetActFeedCB implements IActivityFeedCallback {
@Override
public void receiveResult(MarshaledActivityFeed activityFeedObject) {
System.out.println("in receiveresult: " + activityFeedObject.getGetActivitiesResponse().getMarshaledActivity().size());
for (org.societies.api.schema.activity.MarshaledActivity act : activityFeedObject.getGetActivitiesResponse().getMarshaledActivity()) {
actDiff.add(new Activity(act));
}
cpa.init();
cpa.analyze(actDiff);
}
}
GetActFeedCB dummyFeedback = new GetActFeedCB();
cises.get(0).getActivityFeed().getActivities("0 " + Long.toString(System.currentTimeMillis() + 100000L), maxMem, dummyFeedback);
System.out.println("cises.get(0).getActivityFeed(): " + cises.get(0).getActivityFeed());
return cpa.getGraph().toJung();
}Example 43
| Project: swagger-inflector-master File: DefaultConverter.java View source code |
public Object cast(List<String> o, Parameter parameter, JavaType javaType, Map<String, Model> definitions) throws ConversionException {
if (o == null || o.size() == 0) {
return null;
}
Class<?> cls = javaType.getRawClass();
LOGGER.debug("converting array `" + o + "` to `" + cls + "`");
if (javaType.isArrayType()) {
if (parameter instanceof SerializableParameter) {
List<Object> output = new ArrayList<Object>();
SerializableParameter sp = (SerializableParameter) parameter;
if (sp.getItems() != null) {
if (sp.getItems() instanceof ArrayProperty) {
Property inner = ((ArrayProperty) sp.getItems()).getItems();
// TODO: this does not need to be done this way, update the helper method
Parameter innerParam = new QueryParameter().property(inner);
JavaType innerClass = getTypeFromParameter(innerParam, definitions);
for (String obj : o) {
String[] parts = new String[0];
CSVFormat format = null;
if ("csv".equals(sp.getCollectionFormat()) && !StringUtils.isEmpty(obj)) {
format = CSVFormat.DEFAULT;
} else if ("pipes".equals(sp.getCollectionFormat()) && !StringUtils.isEmpty(obj)) {
format = CSVFormat.newFormat('|').withQuote('"');
} else if ("ssv".equals(sp.getCollectionFormat()) && !StringUtils.isEmpty(obj)) {
format = CSVFormat.newFormat(' ').withQuote('"');
}
if (format != null) {
try {
for (CSVRecord record : CSVParser.parse(obj, format).getRecords()) {
List<String> it = new ArrayList<String>();
for (Iterator<String> x = record.iterator(); x.hasNext(); ) {
it.add(x.next());
}
parts = it.toArray(new String[it.size()]);
}
} catch (IOException e) {
}
} else {
parts = new String[1];
parts[0] = obj;
}
for (String p : parts) {
Object ob = cast(p, inner, innerClass);
if (ob != null) {
output.add(ob);
}
}
}
}
}
return output;
}
} else if (parameter instanceof SerializableParameter) {
SerializableParameter sp = (SerializableParameter) parameter;
return cast(o.get(0), sp.getItems(), javaType);
}
return null;
}Example 44
| Project: tika-master File: ISATabUtils.java View source code |
public static void parseStudy(InputStream stream, XHTMLContentHandler xhtml, Metadata metadata, ParseContext context) throws IOException, TikaException, SAXException {
TikaInputStream tis = TikaInputStream.get(stream);
// Automatically detect the character encoding
TikaConfig tikaConfig = context.get(TikaConfig.class);
if (tikaConfig == null) {
tikaConfig = TikaConfig.getDefaultConfig();
}
try (AutoDetectReader reader = new AutoDetectReader(new CloseShieldInputStream(tis), metadata, tikaConfig.getEncodingDetector());
CSVParser csvParser = new CSVParser(reader, CSVFormat.TDF)) {
Iterator<CSVRecord> iterator = csvParser.iterator();
xhtml.startElement("table");
xhtml.startElement("thead");
if (iterator.hasNext()) {
CSVRecord record = iterator.next();
for (int i = 0; i < record.size(); i++) {
xhtml.startElement("th");
xhtml.characters(record.get(i));
xhtml.endElement("th");
}
}
xhtml.endElement("thead");
xhtml.startElement("tbody");
while (iterator.hasNext()) {
CSVRecord record = iterator.next();
xhtml.startElement("tr");
for (int j = 0; j < record.size(); j++) {
xhtml.startElement("td");
xhtml.characters(record.get(j));
xhtml.endElement("td");
}
xhtml.endElement("tr");
}
xhtml.endElement("tbody");
xhtml.endElement("table");
}
}Example 45
| Project: TradeTrax-master File: Importer.java View source code |
public Importer onSuccessFromCsvForm() {
CSVParser parser = null;
parsed = new Vector<Stock>();
try {
parser = new CSVParser(new StringReader(rawcsv), CSVFormat.EXCEL.withHeader());
for (CSVRecord rec : parser) {
Stock stock = recordToStock(rec);
if (stock != null) {
parsed.add(stock);
}
}
} catch (Exception e) {
}
try {
parser.close();
} catch (Exception e) {
}
return this;
}Example 46
| Project: TundraCSV.java-master File: IDataCSVParser.java View source code |
/**
* Returns an IData representation of the CSV data in the given input stream.
*
* @param inputStream The input stream to be decoded.
* @param charset The character set to use.
* @return An IData representation of the given input stream data.
* @throws IOException If there is a problem reading from the stream.
*/
@Override
public IData decode(InputStream inputStream, Charset charset) throws IOException {
if (inputStream == null)
return null;
Reader reader = new InputStreamReader(inputStream, CharsetHelper.normalize(charset));
CSVFormat format = CSVFormat.DEFAULT.withHeader().withDelimiter(delimiter).withNullString("");
CSVParser parser = format.parse(reader);
Set<String> keys = parser.getHeaderMap().keySet();
List<IData> list = new ArrayList<IData>();
for (CSVRecord record : parser) {
IData document = IDataFactory.create();
IDataCursor cursor = document.getCursor();
for (String key : keys) {
if (record.isSet(key)) {
String value = record.get(key);
if (value != null)
IDataUtil.put(cursor, key, value);
}
}
cursor.destroy();
list.add(document);
}
IData output = IDataFactory.create();
IDataCursor cursor = output.getCursor();
IDataUtil.put(cursor, "recordWithNoID", list.toArray(new IData[list.size()]));
return output;
}Example 47
| Project: Web-Karma-master File: PatternReader.java View source code |
public static Map<String, Pattern> importPatterns(String inputDir, Integer length) {
Map<String, Pattern> patterns = new HashMap<String, Pattern>();
// CSVFormat csvFileFormat = CSVFormat.DEFAULT.withRecordSeparator("\n").
// withIgnoreEmptyLines(true).
// withDelimiter(' ').
// withIgnoreSurroundingSpaces(true).
// withHeader();
CSVFormat csvFileFormat = CSVFormat.RFC4180.withHeader();
CSVParser csvParserTotalFrequency;
CSVParser csvParser;
File patternDir = new File(inputDir);
if (patternDir.exists()) {
File[] patternFiles = patternDir.listFiles();
for (File file : patternFiles) {
try {
if (file.isDirectory())
continue;
csvParser = CSVParser.parse(file, Charsets.UTF_8, csvFileFormat);
Map<String, Integer> headerMap = csvParser.getHeaderMap();
csvParserTotalFrequency = CSVParser.parse(file, Charsets.UTF_8, csvFileFormat);
int totalFrequency = getTotalFrequency(csvParserTotalFrequency, headerMap.get(COUNT_COLUMN));
for (CSVRecord record : csvParser) {
Pattern p = getPattern(record, headerMap, totalFrequency);
if (p != null) {
if (length != null) {
if (p.getLength() == length.intValue())
patterns.put(p.getId(), p);
} else {
patterns.put(p.getId(), p);
}
// System.out.println(p.getPrintStr());
} else {
logger.error("Error in reading the pattern: " + record.toString());
}
}
} catch (IOException e) {
logger.error("Error in reading the patterns from the file " + file.getAbsolutePath());
e.printStackTrace();
}
}
}
return patterns;
}Example 48
| Project: bw-calendar-engine-master File: ProcessCreate.java View source code |
private boolean createLocationsCsv() throws Throwable {
try {
open();
final String fileName = quotedVal();
if (fileName == null) {
error("Expected csv filename");
return false;
}
final File csvf = new File(fileName);
if (!csvf.exists()) {
error("File " + fileName + " does not exist");
return false;
}
final CSVParser parser = CSVFormat.DEFAULT.withHeader().withAllowMissingColumnNames().parse(new FileReader(csvf));
for (final CSVRecord rec : parser) {
final BwLocation loc = BwLocation.makeLocation();
String fld = rec.get("Address");
if (fld == null) {
error("address required for location");
continue;
}
loc.setAddressField(fld);
fld = rec.get("Room");
if (fld != null) {
loc.setRoomField(fld);
}
fld = rec.get("Code");
if (fld != null) {
loc.setSubField1(fld);
}
/* fld = rec.get("subField2");
if (fld != null) {
loc.setSubField1(fld);
}*/
fld = rec.get("Accessible");
if (fld != null) {
loc.setAccessible("Y".equals(fld));
}
fld = rec.get("Geouri");
if (fld != null) {
loc.setGeouri(fld);
}
fld = rec.get("Street");
if (fld != null) {
loc.setStreet(fld);
}
fld = rec.get("City");
if (fld != null) {
loc.setCity(fld);
}
fld = rec.get("State");
if (fld != null) {
loc.setState(fld);
}
fld = rec.get("Zip");
if (fld != null) {
loc.setZip(fld);
}
fld = rec.get("Link");
if (fld != null) {
loc.setLink(fld);
}
getSvci().getLocationsHandler().add(loc);
}
return true;
} finally {
close();
}
}Example 49
| Project: geowave-master File: SceneFeatureIterator.java View source code |
private void setupCsvToFeatureIterator(final File csvFile, final long startLine, final WRS2GeometryStore geometryStore, final Filter cqlFilter) throws FileNotFoundException, IOException {
parserFis = new FileInputStream(csvFile);
parserIsr = new InputStreamReader(parserFis, StringUtils.UTF8_CHAR_SET);
parser = new CSVParser(parserIsr, CSVFormat.DEFAULT.withHeader().withSkipHeaderRecord());
final Iterator<CSVRecord> csvIterator = parser.iterator();
long startLineDecrementor = startLine;
// we skip the header, so only skip to start line 1
while ((startLineDecrementor > 1) && csvIterator.hasNext()) {
startLineDecrementor--;
csvIterator.next();
}
// wrap the iterator with a feature conversion and a filter (if
// provided)
iterator = Iterators.transform(csvIterator, new CSVToFeatureTransform(geometryStore, type));
if (cqlFilter != null) {
Filter actualFilter;
if (hasOtherProperties(cqlFilter)) {
final PropertyIgnoringFilterVisitor visitor = new PropertyIgnoringFilterVisitor(SCENE_ATTRIBUTES, type);
actualFilter = (Filter) cqlFilter.accept(visitor, null);
} else {
actualFilter = cqlFilter;
}
final CqlFilterPredicate filterPredicate = new CqlFilterPredicate(actualFilter);
iterator = Iterators.filter(iterator, filterPredicate);
}
}Example 50
| Project: mdrill-master File: CSVRequestHandler.java View source code |
@Override
void add(SolrInputDocument doc, int line, int column, String val) {
CSVParser parser = new CSVParser(new StringReader(val), strategy);
try {
String[] vals = parser.getLine();
if (vals != null) {
for (String v : vals) base.add(doc, line, column, v);
} else {
base.add(doc, line, column, val);
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}Example 51
| Project: opencast-master File: EventsLoader.java View source code |
private void execute(CSVParser csv) throws Exception {
List<EventEntry> events = parseCSV(csv);
logger.info("Found {} events to populate", events.size());
int i = 1;
final Date now = new Date();
for (EventEntry event : events) {
logger.info("Populating event {}", i);
createSeries(event);
Tuple<MediaPackage, DublinCoreCatalog> basicMediaPackage = getBasicMediaPackage(event);
if (now.after(event.getRecordingDate())) {
addWorkflowEntry(basicMediaPackage.getA());
if (event.isArchive())
addArchiveEntry(basicMediaPackage.getA());
} else {
addSchedulerEntry(event, basicMediaPackage.getB());
}
logger.info("Finished populating event {}", i++);
}
}Example 52
| Project: Solbase-Solr-master File: CSVRequestHandler.java View source code |
void add(SolrInputDocument doc, int line, int column, String val) {
CSVParser parser = new CSVParser(new StringReader(val), strategy);
try {
String[] vals = parser.getLine();
if (vals != null) {
for (String v : vals) base.add(doc, line, column, v);
} else {
base.add(doc, line, column, val);
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}Example 53
| Project: solrcene-master File: CSVRequestHandler.java View source code |
void add(SolrInputDocument doc, int line, int column, String val) {
CSVParser parser = new CSVParser(new StringReader(val), strategy);
try {
String[] vals = parser.getLine();
if (vals != null) {
for (String v : vals) base.add(doc, line, column, v);
} else {
base.add(doc, line, column, val);
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}Example 54
| Project: tradelib-master File: Series.java View source code |
public static Series fromCsv(String path, boolean header, DateTimeFormatter dtf, LocalTime lt) throws Exception {
if (dtf == null) {
if (lt == null)
dtf = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
else
dtf = DateTimeFormatter.ISO_DATE;
}
// Parse and import the csv
CSVFormat csvFmt = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreSurroundingSpaces();
if (header)
csvFmt = csvFmt.withHeader();
CSVParser csv = csvFmt.parse(new BufferedReader(new FileReader(path)));
int ncols = -1;
Series result = null;
double[] values = null;
for (CSVRecord rec : csv.getRecords()) {
if (result == null) {
ncols = rec.size() - 1;
values = new double[ncols];
result = new Series(ncols);
}
for (int ii = 0; ii < ncols; ++ii) {
values[ii] = Double.parseDouble(rec.get(ii + 1));
}
LocalDateTime ldt;
if (lt != null) {
ldt = LocalDate.parse(rec.get(0), dtf).atTime(lt);
} else {
ldt = LocalDateTime.parse(rec.get(0), dtf);
}
result.append(ldt, values);
}
if (header) {
Map<String, Integer> headerMap = csv.getHeaderMap();
result.clearNames();
for (Map.Entry<String, Integer> me : headerMap.entrySet()) {
if (me.getValue() > 0)
result.setName(me.getKey(), me.getValue() - 1);
}
}
return result;
}Example 55
| Project: vitam-master File: RulesManagerFileImpl.java View source code |
@Override
public ArrayNode checkFile(InputStream rulesFileStream) throws IOException, ReferentialException, InvalidParseOperationException {
ParametersChecker.checkParameter(RULES_FILE_STREAM_IS_A_MANDATORY_PARAMETER, rulesFileStream);
if (!isCollectionEmptyForTenant()) {
throw new FileRulesException(String.format(RULES_COLLECTION_IS_NOT_EMPTY, getTenant()));
}
File csvFileReader = convertInputStreamToFile(rulesFileStream);
try (FileReader reader = new FileReader(csvFileReader)) {
@SuppressWarnings("resource") final CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.withHeader());
final HashSet<String> ruleIdSet = new HashSet<>();
for (final CSVRecord record : parser) {
try {
if (checkRecords(record)) {
final String ruleId = record.get(RULEID);
final String ruleType = record.get(RULE_TYPE);
final String ruleValue = record.get(RULE_VALUE);
final String ruleDuration = record.get(RULE_DURATION);
final String ruleMeasurementValue = record.get(RULE_MEASUREMENT);
checkParametersNotEmpty(ruleId, ruleType, ruleValue, ruleDuration, ruleMeasurementValue);
checkRuleDurationIsInteger(ruleDuration);
if (ruleIdSet.contains(ruleId)) {
throw new FileRulesException(String.format(FILE_RULE_WITH_RULE_ID, ruleId));
}
ruleIdSet.add(ruleId);
if (!contains(ruleMeasurementValue)) {
throw new FileRulesException(String.format(NOT_SUPPORTED_VALUE, RULE_MEASUREMENT, ruleMeasurementValue));
}
}
} catch (final Exception e) {
throw new FileRulesException(INVALID_CSV_FILE + e.getMessage());
}
}
}
if (csvFileReader != null) {
final ArrayNode readRulesAsJson = RulesManagerParser.readObjectsFromCsvWriteAsArrayNode(csvFileReader);
csvFileReader.delete();
return readRulesAsJson;
}
/* this line is reached only if temporary file is null */
throw new FileRulesException(INVALID_CSV_FILE);
}Example 56
| Project: alma-toolkit-master File: TaskUpdateResourcePartners.java View source code |
public ConcurrentMap<String, Partner> getTepunaPartners() {
String prefix = "NLNZ";
String institutionCode = config.getProperty("ladd.institution.code");
ConcurrentMap<String, Partner> result = new ConcurrentHashMap<String, Partner>();
WebClient webClient = webClientProvider.get();
TextPage page = null;
try {
log.debug("tepuna url: {}", tepunaUrl);
page = webClient.getPage(tepunaUrl);
} catch (IOException e) {
log.error("unable to acquire page: {}", tepunaUrl);
return result;
}
log.debug("{}", page.getContent());
try (CSVParser parser = CSVParser.parse(page.getContent(), CSVFormat.DEFAULT.withHeader())) {
for (CSVRecord record : parser) {
String nuc = record.get(0);
if ("NUC symbol".equals(nuc) || institutionCode.equals(nuc)) {
log.debug("skipping nuc: {}", nuc);
continue;
}
nuc = prefix + ":" + nuc;
String org = record.get(2);
Partner partner = new Partner();
partner.setLink("https://api-ap.hosted.exlibrisgroup.com/almaws/v1/partners/" + nuc);
PartnerDetails partnerDetails = new PartnerDetails();
partner.setPartnerDetails(partnerDetails);
ProfileDetails profileDetails = new ProfileDetails();
partnerDetails.setProfileDetails(profileDetails);
profileDetails.setProfileType(ProfileType.ISO);
RequestExpiryType requestExpiryType = new RequestExpiryType();
requestExpiryType.setValue("INTEREST_DATE");
requestExpiryType.setDesc("Expire by interest date");
IsoDetails isoDetails = new IsoDetails();
profileDetails.setIsoDetails(isoDetails);
isoDetails.setAlternativeDocumentDelivery(false);
isoDetails.setIllServer(config.getProperty("alma.ill.server"));
isoDetails.setIllPort(Integer.parseInt(config.getProperty("alma.ill.port")));
isoDetails.setIsoSymbol(nuc);
isoDetails.setSendRequesterInformation(false);
isoDetails.setSharedBarcodes(true);
isoDetails.setRequestExpiryType(requestExpiryType);
SystemType systemType = new SystemType();
systemType.setValue("LADD");
systemType.setDesc("LADD");
LocateProfile locateProfile = new LocateProfile();
locateProfile.setValue("LADD");
locateProfile.setDesc("LADD Locate Profile");
partnerDetails.setStatus(Status.ACTIVE);
partnerDetails.setCode(nuc);
partnerDetails.setName(org);
partnerDetails.setSystemType(systemType);
partnerDetails.setAvgSupplyTime(4);
partnerDetails.setDeliveryDelay(4);
partnerDetails.setCurrency("AUD");
partnerDetails.setBorrowingSupported(true);
partnerDetails.setBorrowingWorkflow("LADD_Borrowing");
partnerDetails.setLendingSupported(true);
partnerDetails.setLendingWorkflow("LADD_Lending");
partnerDetails.setLocateProfile(locateProfile);
partnerDetails.setHoldingCode(nuc);
ContactInfo contactInfo = new ContactInfo();
partner.setContactInfo(contactInfo);
Addresses addresses = new Addresses();
contactInfo.setAddresses(addresses);
String s = record.get(5);
if (s == null || "".equals(s.trim()))
s = record.get(4);
if (s != null && !"".equals(s.trim())) {
Address address = getAddress(s);
address.setPreferred(true);
address.setAddressTypes(new AddressTypes());
address.getAddressTypes().getAddressType().add("ALL");
addresses.getAddress().add(address);
log.debug("nuc/address [{}]: {}", nuc, address);
}
Emails emails = new Emails();
contactInfo.setEmails(emails);
s = record.get(6);
if (s != null && !"".equals(s.trim())) {
Email email = new Email();
email.setEmailTypes(new EmailTypes());
email.setEmailAddress(s);
email.setPreferred(true);
email.setDescription("Primary Email Address");
email.getEmailTypes().getEmailType().add("ALL");
emails.getEmail().add(email);
log.debug("nuc/email1 [{}]: {}", nuc, email);
}
s = record.get(13);
if (s != null && !"".equals(s.trim())) {
Email email = new Email();
email.setEmailTypes(new EmailTypes());
email.setEmailAddress(s);
email.setPreferred(true);
String m = record.get(12);
if (m != null && !"".equals(m))
email.setDescription("Manager Email Address: " + m);
else
email.setDescription("Manager Email Address");
email.getEmailTypes().getEmailType().add("ALL");
emails.getEmail().add(email);
log.debug("nuc/email2 [{}]: {}", nuc, email);
}
Phones phones = new Phones();
contactInfo.setPhones(phones);
s = record.get(15);
if (s == null || "".equals(s.trim()))
s = record.get(7);
if (s != null && !"".equals(s.trim())) {
Phone phone = new Phone();
phone.setPhoneTypes(new PhoneTypes());
phone.setPhoneNumber(s);
phone.setPreferred(true);
phone.setPreferredSMS(false);
phone.getPhoneTypes().getPhoneType().add("ALL");
phones.getPhone().add(phone);
log.debug("nuc/phone [{}]: {}", nuc, phone);
}
Notes notes = new Notes();
partner.setNotes(notes);
result.put(partner.getPartnerDetails().getCode(), partner);
}
} catch (IOException ioe) {
log.error("unable to parse data: {}", tepunaUrl);
}
return result;
}Example 57
| Project: hapi-fhir-master File: TerminologyLoaderSvc.java View source code |
private void iterateOverZipFile(List<byte[]> theZipBytes, String fileNamePart, IRecordHandler handler, char theDelimiter, QuoteMode theQuoteMode) {
boolean found = false;
for (byte[] nextZipBytes : theZipBytes) {
ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new ByteArrayInputStream(nextZipBytes)));
try {
for (ZipEntry nextEntry; (nextEntry = zis.getNextEntry()) != null; ) {
ZippedFileInputStream inputStream = new ZippedFileInputStream(zis);
String nextFilename = nextEntry.getName();
if (nextFilename.contains(fileNamePart)) {
ourLog.info("Processing file {}", nextFilename);
found = true;
Reader reader = null;
CSVParser parsed = null;
try {
reader = new InputStreamReader(new BOMInputStream(zis), Charsets.UTF_8);
CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
if (theQuoteMode != null) {
format = format.withQuote('"').withQuoteMode(theQuoteMode);
}
parsed = new CSVParser(reader, format);
Iterator<CSVRecord> iter = parsed.iterator();
ourLog.debug("Header map: {}", parsed.getHeaderMap());
int count = 0;
int logIncrement = LOG_INCREMENT;
int nextLoggedCount = 0;
while (iter.hasNext()) {
CSVRecord nextRecord = iter.next();
handler.accept(nextRecord);
count++;
if (count >= nextLoggedCount) {
ourLog.info(" * Processed {} records in {}", count, nextFilename);
nextLoggedCount += logIncrement;
}
}
} catch (IOException e) {
throw new InternalErrorException(e);
}
}
}
} catch (IOException e) {
throw new InternalErrorException(e);
} finally {
IOUtils.closeQuietly(zis);
}
}
// This should always be true, but just in case we've introduced a bug...
Validate.isTrue(found);
}Example 58
| Project: rce-master File: DOESection.java View source code |
private boolean loadTableFromFile(String path) {
boolean success = true;
try {
File csvData = new File(path);
CSVParser parser = CSVParser.parse(FileUtils.readFileToString(csvData), CSVFormat.newFormat(';').withIgnoreSurroundingSpaces().withAllowMissingColumnNames().withRecordSeparator("\n"));
List<CSVRecord> records = parser.getRecords();
if (Character.isLetter((records.get(0).get(0).charAt(0)))) {
records.remove(0);
}
String[][] values = new String[records.size()][];
int count = 0;
for (CSVRecord record : records) {
int size = getOutputs().size();
values[count] = new String[size];
for (int i = 0; i < size; i++) {
if (record.size() > i) {
String number = record.get(i);
if (number.equals("null")) {
values[count][i] = "";
} else {
Matcher matcher = Pattern.compile("(\\+|-)?\\d+(,|.)?\\d*(e|E)?(\\+|-)?\\d*").matcher(number);
if (count == 0 && i == 0 && matcher.find()) {
number = matcher.group();
}
values[count][i] = number.replaceAll("\"", " ").replaceAll(",", ".");
}
} else {
values[count][i] = "";
}
}
count++;
}
ObjectMapper mapper = JsonUtils.getDefaultObjectMapper();
setProperty(DOEConstants.KEY_TABLE, mapper.writeValueAsString(values));
setProperty(DOEConstants.KEY_START_SAMPLE, "0");
setProperty(DOEConstants.KEY_END_SAMPLE, Integer.toString(count - 1));
setProperty(DOEConstants.KEY_RUN_NUMBER, Integer.toString(count));
setProperty(DOEConstants.KEY_METHOD, DOEConstants.DOE_ALGORITHM_CUSTOM_TABLE);
algorithmSelection.select(algorithmSelection.indexOf(DOEConstants.DOE_ALGORITHM_CUSTOM_TABLE));
refreshSection();
if (values[0] != null && values[0].length > getOutputs().size()) {
MessageDialog.openInformation(this.getComposite().getShell(), "Warning", "There are more values per run in the loaded table than outputs defined.");
}
} catch (IOException e) {
LOGGER.error(e);
success = false;
if (!success) {
MessageDialog.openInformation(this.getComposite().getShell(), ERROR, "Could not load table from file.\n\nReason: " + e.getMessage());
}
} catch (NumberFormatException e) {
LOGGER.error(e);
success = false;
if (!success) {
MessageDialog.openInformation(this.getComposite().getShell(), ERROR, "Could not load table because of a syntax error.\n\nReason: " + e.getMessage());
}
}
return success;
}Example 59
| Project: datumbox-framework-master File: Dataframe.java View source code |
/**
* It builds a Dataframe object from a CSV file; the first line of the provided
* CSV file must have a header with the column names.
*
* The method accepts the following arguments: A Reader object from where
* we will read the contents of the csv file. The name column of the
* response variable y. A map with the column names and their respective
* DataTypes. The char delimiter for the columns, the char for quotes and
* the string of the record/row separator. The Storage Configuration
* object.
*
* @param reader
* @param yVariable
* @param headerDataTypes
* @param delimiter
* @param quote
* @param recordSeparator
* @param skip
* @param limit
* @param configuration
* @return
*/
public static Dataframe parseCSVFile(Reader reader, String yVariable, LinkedHashMap<String, TypeInference.DataType> headerDataTypes, char delimiter, char quote, String recordSeparator, Long skip, Long limit, Configuration configuration) {
Logger logger = LoggerFactory.getLogger(Dataframe.Builder.class);
if (skip == null) {
skip = 0L;
}
if (limit == null) {
limit = Long.MAX_VALUE;
}
logger.info("Parsing CSV file");
if (!headerDataTypes.containsKey(yVariable)) {
logger.warn("WARNING: The file is missing the response variable column {}.", yVariable);
}
TypeInference.DataType yDataType = headerDataTypes.get(yVariable);
//copy header types
Map<String, TypeInference.DataType> xDataTypes = new HashMap<>(headerDataTypes);
//remove the response variable from xDataTypes
xDataTypes.remove(yVariable);
//use the private constructor to pass DataTypes directly and avoid updating them on the fly
Dataframe dataset = new Dataframe(configuration, yDataType, xDataTypes);
CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(delimiter).withQuote(quote).withRecordSeparator(recordSeparator);
try (final CSVParser parser = new CSVParser(reader, format)) {
ThreadMethods.throttledExecution(StreamMethods.enumerate(StreamMethods.stream(parser.spliterator(), false)).skip(skip).limit(limit), e -> {
Integer rId = e.getKey();
CSVRecord row = e.getValue();
if (!row.isConsistent()) {
logger.warn("WARNING: Skipping row {} because its size does not match the header size.", row.getRecordNumber());
} else {
Object y = null;
AssociativeArray xData = new AssociativeArray();
for (Map.Entry<String, TypeInference.DataType> entry : headerDataTypes.entrySet()) {
String column = entry.getKey();
TypeInference.DataType dataType = entry.getValue();
Object value = TypeInference.DataType.parse(row.get(column), dataType);
if (yVariable != null && yVariable.equals(column)) {
y = value;
} else {
xData.put(column, value);
}
}
Record r = new Record(xData, y);
dataset._unsafe_set(rId, r);
}
}, configuration.getConcurrencyConfiguration());
} catch (IOException ex) {
throw new RuntimeException(ex);
}
return dataset;
}Example 60
| Project: jena-master File: CSVParser.java View source code |
public static CSVParser create(String filename) {
InputStream input = IO.openFile(filename);
return create(input);
}Example 61
| Project: easybatch-framework-master File: ApacheCommonCsvRecordMapper.java View source code |
@Override
public Record<P> processRecord(final StringRecord record) throws Exception {
String payload = record.getPayload();
CSVParser csvParser = csvFormat.parse(new StringReader(payload));
CSVRecord csvRecord = csvParser.iterator().next();
csvParser.close();
return new GenericRecord<>(record.getHeader(), objectMapper.mapObject(csvRecord.toMap()));
}Example 62
| Project: pinot-master File: CSVRecordReader.java View source code |
@Override
public void init() throws Exception {
final Reader reader = new FileReader(_fileName);
_parser = new CSVParser(reader, getFormat());
_iterator = _parser.iterator();
}Example 63
| Project: geosummly-master File: CSVDataIO.java View source code |
/**
* Get a csv file as a List of CSVRecord.
*/
public List<CSVRecord> readCSVFile(String file) throws IOException {
FileReader reader = new FileReader(file);
CSVParser parser = new CSVParser(reader, CSVFormat.EXCEL);
List<CSVRecord> list = parser.getRecords();
parser.close();
return list;
}Example 64
| Project: orientdb-master File: OCSVExtractor.java View source code |
@Override
public void extract(final Reader iReader) {
super.extract(iReader);
try {
CSVParser parser = new CSVParser(iReader, csvFormat);
recordIterator = parser.iterator();
} catch (IOException e) {
throw new OExtractorException(e);
}
}Example 65
| Project: testsuite-nocoding-master File: CSVBasedURLActionDataListBuilder.java View source code |
private CSVParser createCSVParserFromFilepath(final String filePath) { final BufferedReader br = createBufferedReaderFromFile(new File(filePath)); final CSVParser parser = createCSVParser(br, CSV_FORMAT); return parser; }