/*
* The Kuali Financial System, a comprehensive financial management system for higher education.
*
* Copyright 2005-2014 The Kuali Foundation
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.kuali.kfs.module.cg.service.impl;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Calendar;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPReply;
import org.kuali.kfs.module.cg.batch.CfdaBatchStep;
import org.kuali.kfs.module.cg.businessobject.CFDA;
import org.kuali.kfs.module.cg.businessobject.CfdaUpdateResults;
import org.kuali.kfs.module.cg.service.CfdaService;
import org.kuali.kfs.sys.KFSConstants;
import org.kuali.kfs.sys.service.NonTransactional;
import org.kuali.rice.core.api.datetime.DateTimeService;
import org.kuali.rice.coreservice.framework.parameter.ParameterService;
import org.kuali.rice.krad.service.BusinessObjectService;
import org.springframework.transaction.annotation.Transactional;
import au.com.bytecode.opencsv.CSVReader;
public class CfdaServiceImpl implements CfdaService {
private static final org.apache.log4j.Logger LOG = org.apache.log4j.Logger.getLogger(CfdaServiceImpl.class);
protected BusinessObjectService businessObjectService;
protected static Comparator cfdaComparator;
private DateTimeService dateTimeService;
protected ParameterService parameterService;
static {
cfdaComparator = new Comparator() {
@Override
public int compare(Object o1, Object o2) {
String lhs = (String) o1;
String rhs = (String) o2;
return lhs.compareTo(rhs);
}
};
}
@NonTransactional
public void setDateTimeService(DateTimeService dateTimeService) {
this.dateTimeService = dateTimeService;
}
/**
* Sets the parameterService attribute value.
*
* @param parameterService The parameterService to set.
*/
public void setParameterService(ParameterService parameterService) {
this.parameterService = parameterService;
}
/**
* @return
* @throws IOException
*/
public SortedMap<String, CFDA> getGovCodes() throws IOException {
Calendar calendar = dateTimeService.getCurrentCalendar();
SortedMap<String, CFDA> govMap = new TreeMap<String, CFDA>();
// ftp://ftp.cfda.gov/programs09187.csv
String govURL = parameterService.getParameterValueAsString(CfdaBatchStep.class, KFSConstants.SOURCE_URL_PARAMETER);
String fileName = StringUtils.substringAfterLast(govURL, "/");
govURL = StringUtils.substringBeforeLast(govURL, "/");
if (StringUtils.contains(govURL, "ftp://")) {
govURL = StringUtils.remove(govURL, "ftp://");
}
// need to pull off the '20' in 2009
String year = "" + calendar.get(Calendar.YEAR);
year = year.substring(2, 4);
fileName = fileName + year;
// the last 3 numbers in the file name are the day of the year, but the files are from "yesterday"
fileName = fileName + String.format("%03d", calendar.get(Calendar.DAY_OF_YEAR) - 1);
fileName = fileName + ".csv";
LOG.info("Getting government file: " + fileName + " for update");
InputStream inputStream = null;
FTPClient ftp = new FTPClient();
try {
ftp.connect(govURL);
int reply = ftp.getReplyCode();
if (!FTPReply.isPositiveCompletion(reply)) {
LOG.error("FTP connection to server not established.");
throw new IOException("FTP connection to server not established.");
}
boolean isLoggedIn = ftp.login("anonymous", "");
if (!isLoggedIn) {
LOG.error("Could not login as anonymous.");
throw new IOException("Could not login as anonymous.");
}
LOG.info("Successfully connected and logged in");
ftp.enterLocalPassiveMode();
inputStream = ftp.retrieveFileStream(fileName);
if (inputStream != null) {
LOG.info("reading input stream");
InputStreamReader screenReader = new InputStreamReader(inputStream);
BufferedReader screen = new BufferedReader(screenReader);
CSVReader csvReader = new CSVReader(screenReader, ',', '"', 1);
List<String[]> lines = csvReader.readAll();
for (String[] line : lines) {
String title = line[0];
String number = line[1];
CFDA cfda = new CFDA();
cfda.setCfdaNumber(number);
cfda.setCfdaProgramTitleName(title);
govMap.put(number, cfda);
}
}
ftp.logout();
ftp.disconnect();
}
finally {
if (ftp.isConnected()) {
ftp.disconnect();
}
}
return govMap;
}
/**
* @return
* @throws IOException
*/
public SortedMap<String, CFDA> getKfsCodes() throws IOException {
Collection allCodes = businessObjectService.findAll(CFDA.class);
SortedMap<String, CFDA> kfsMapAll = new TreeMap<String, CFDA>(cfdaComparator);
for (Object o : allCodes) {
CFDA c = (CFDA) o;
kfsMapAll.put(c.getCfdaNumber(), c);
}
return kfsMapAll;
}
/**
* @see org.kuali.kfs.module.cg.service.CfdaService#update()
*/
@Transactional
@Override
public CfdaUpdateResults update() throws IOException {
CfdaUpdateResults results = new CfdaUpdateResults();
Map<String, CFDA> govMap = null;
try {
govMap = getGovCodes();
}
catch (IOException ioe) {
LOG.error("Error connecting to URL resource: " + ioe.getMessage(), ioe);
StringBuilder builder = new StringBuilder();
builder.append("No updates took place.\n");
builder.append(ioe.getMessage());
results.setMessage(builder.toString());
return results;
}
Map<String, CFDA> kfsMap = getKfsCodes();
results.setNumberOfRecordsInKfsDatabase(kfsMap.keySet().size());
results.setNumberOfRecordsRetrievedFromWebSite(govMap.keySet().size());
for (Object key : kfsMap.keySet()) {
CFDA cfdaKfs = kfsMap.get(key);
CFDA cfdaGov = govMap.get(key);
if (cfdaKfs.getCfdaMaintenanceTypeId().startsWith("M")) {
// Leave it alone. It's maintained manually.
results.setNumberOfRecordsNotUpdatedBecauseManual(1 + results.getNumberOfRecordsNotUpdatedBecauseManual());
}
else if (cfdaKfs.getCfdaMaintenanceTypeId().startsWith("A")) {
if (cfdaGov == null) {
if (cfdaKfs.isActive()) {
cfdaKfs.setActive(false);
businessObjectService.save(cfdaKfs);
results.setNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite(results.getNumberOfRecordsDeactivatedBecauseNoLongerOnWebSite() + 1);
}
else {
// Leave it alone for historical purposes
results.setNumberOfRecrodsNotUpdatedForHistoricalPurposes(results.getNumberOfRecrodsNotUpdatedForHistoricalPurposes() + 1);
}
}
else {
if (cfdaKfs.isActive()) {
results.setNumberOfRecordsUpdatedBecauseAutomatic(results.getNumberOfRecordsUpdatedBecauseAutomatic() + 1);
}
else {
cfdaKfs.setActive(true);
results.setNumberOfRecordsReActivated(results.getNumberOfRecordsReActivated() + 1);
}
cfdaKfs.setCfdaProgramTitleName(cfdaGov.getCfdaProgramTitleName());
businessObjectService.save(cfdaKfs);
}
}
// Remove it from the govMap so we know what codes from the govMap don't already exist in KFS.
govMap.remove(key);
}
// What's left in govMap now is just the codes that don't exist in KFS
for (String key : govMap.keySet()) {
CFDA cfdaGov = govMap.get(key);
cfdaGov.setCfdaMaintenanceTypeId("AUTOMATIC");
cfdaGov.setActive(true);
businessObjectService.save(cfdaGov);
results.setNumberOfRecordsNewlyAddedFromWebSite(results.getNumberOfRecordsNewlyAddedFromWebSite() + 1);
}
return results;
}
public void setBusinessObjectService(BusinessObjectService businessObjectService) {
this.businessObjectService = businessObjectService;
}
@Override
public CFDA getByPrimaryId(String cfdaNumber) {
if (StringUtils.isBlank(cfdaNumber)) {
return null;
}
return businessObjectService.findBySinglePrimaryKey(CFDA.class, cfdaNumber.trim());
}
}