/*
This file is part of Cyclos (www.cyclos.org).
A project of the Social Trade Organisation (www.socialtrade.org).
Cyclos is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
Cyclos is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Cyclos; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
package nl.strohalm.cyclos.services.ads;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import nl.strohalm.cyclos.dao.ads.imports.AdImportDAO;
import nl.strohalm.cyclos.dao.ads.imports.ImportedAdCategoryDAO;
import nl.strohalm.cyclos.dao.ads.imports.ImportedAdDAO;
import nl.strohalm.cyclos.entities.Relationship;
import nl.strohalm.cyclos.entities.access.MemberUser;
import nl.strohalm.cyclos.entities.access.User;
import nl.strohalm.cyclos.entities.accounts.Currency;
import nl.strohalm.cyclos.entities.ads.Ad;
import nl.strohalm.cyclos.entities.ads.Ad.TradeType;
import nl.strohalm.cyclos.entities.ads.AdCategory;
import nl.strohalm.cyclos.entities.ads.imports.AdImport;
import nl.strohalm.cyclos.entities.ads.imports.AdImportResult;
import nl.strohalm.cyclos.entities.ads.imports.ImportedAd;
import nl.strohalm.cyclos.entities.ads.imports.ImportedAdCategory;
import nl.strohalm.cyclos.entities.ads.imports.ImportedAdCustomFieldValue;
import nl.strohalm.cyclos.entities.ads.imports.ImportedAdQuery;
import nl.strohalm.cyclos.entities.customization.fields.AdCustomField;
import nl.strohalm.cyclos.entities.customization.fields.AdCustomFieldValue;
import nl.strohalm.cyclos.entities.customization.fields.CustomField;
import nl.strohalm.cyclos.entities.customization.fields.CustomFieldValue;
import nl.strohalm.cyclos.entities.exceptions.EntityNotFoundException;
import nl.strohalm.cyclos.entities.groups.MemberGroupSettings;
import nl.strohalm.cyclos.entities.members.Administrator;
import nl.strohalm.cyclos.entities.members.Element;
import nl.strohalm.cyclos.entities.members.Member;
import nl.strohalm.cyclos.entities.settings.LocalSettings;
import nl.strohalm.cyclos.services.customization.AdCustomFieldServiceLocal;
import nl.strohalm.cyclos.services.elements.ElementServiceLocal;
import nl.strohalm.cyclos.services.fetch.FetchServiceLocal;
import nl.strohalm.cyclos.services.settings.SettingsServiceLocal;
import nl.strohalm.cyclos.utils.CacheCleaner;
import nl.strohalm.cyclos.utils.Period;
import nl.strohalm.cyclos.utils.RelationshipHelper;
import nl.strohalm.cyclos.utils.TimePeriod;
import nl.strohalm.cyclos.utils.access.LoggedUser;
import nl.strohalm.cyclos.utils.conversion.CalendarConverter;
import nl.strohalm.cyclos.utils.conversion.CoercionHelper;
import nl.strohalm.cyclos.utils.conversion.NumberConverter;
import nl.strohalm.cyclos.utils.csv.CSVReader;
import nl.strohalm.cyclos.utils.csv.UnknownColumnException;
import nl.strohalm.cyclos.utils.query.PageHelper;
import nl.strohalm.cyclos.utils.validation.ValidationError;
import nl.strohalm.cyclos.utils.validation.ValidationException;
import nl.strohalm.cyclos.utils.validation.Validator;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
public class AdImportServiceImpl implements AdImportServiceLocal {
private FetchServiceLocal fetchService;
private ElementServiceLocal elementService;
private AdServiceLocal adService;
private AdCategoryServiceLocal adCategoryService;
private SettingsServiceLocal settingsService;
private AdCustomFieldServiceLocal adCustomFieldService;
private AdImportDAO adImportDao;
private ImportedAdDAO importedAdDao;
private ImportedAdCategoryDAO importedAdCategoryDao;
@Override
public List<ImportedAdCategory> getNewCategories(final AdImport adImport) {
return importedAdCategoryDao.getLeafCategories(adImport);
}
@Override
public AdImportResult getSummary(final AdImport adIimport) {
final AdImportResult result = new AdImportResult();
final ImportedAdQuery query = new ImportedAdQuery();
query.setAdImport(adIimport);
query.setPageForCount();
// Get the total number of ads
query.setStatus(ImportedAdQuery.Status.ALL);
result.setTotal(PageHelper.getTotalCount(importedAdDao.search(query)));
// Get the number of ads with error
query.setStatus(ImportedAdQuery.Status.ERROR);
result.setErrors(PageHelper.getTotalCount(importedAdDao.search(query)));
// Get the number of new categories
result.setNewCategories(importedAdCategoryDao.getLeafCategories(adIimport).size());
return result;
}
@Override
public AdImport importAds(AdImport adImport, final InputStream data) {
// Validate and save the import
getValidator().validate(adImport);
final Currency currency = fetchService.fetch(adImport.getCurrency());
adImport.setCurrency(currency);
adImport.setBy(LoggedUser.<Administrator> element());
adImport.setDate(Calendar.getInstance());
adImport = adImportDao.insert(adImport);
// Find out the custom fields
final List<AdCustomField> customFields = adCustomFieldService.list();
final Map<String, CustomField> customFieldMap = new HashMap<String, CustomField>(customFields.size());
for (final AdCustomField customField : customFields) {
customFieldMap.put(customField.getInternalName().toLowerCase(), fetchService.fetch(customField, CustomField.Relationships.POSSIBLE_VALUES));
}
// Find the existing advertisement categories
final Map<String, AdCategory> existingAdCategoryMap = new LinkedHashMap<String, AdCategory>();
final List<AdCategory> rootCategories = adCategoryService.listRoot();
for (final AdCategory adCategory : rootCategories) {
appendCategory(adCategory, existingAdCategoryMap);
}
final Map<String, ImportedAdCategory> importedAdCategoryMap = new LinkedHashMap<String, ImportedAdCategory>();
// Get the settings
final LocalSettings localSettings = settingsService.getLocalSettings();
final char stringQuote = CoercionHelper.coerce(Character.TYPE, localSettings.getCsvStringQuote().getValue());
final char valueSeparator = CoercionHelper.coerce(Character.TYPE, localSettings.getCsvValueSeparator().getValue());
// Read the headers
BufferedReader in = null;
List<String> headers;
try {
in = new BufferedReader(new InputStreamReader(data, localSettings.getCharset()));
headers = CSVReader.readLine(in, stringQuote, valueSeparator);
} catch (final Exception e) {
throw new ValidationException("adImport.invalidFormat");
}
// Import each ad
try {
final CacheCleaner cacheCleaner = new CacheCleaner(fetchService);
int lineNumber = 2; // The first line is the header
List<String> values;
while ((values = CSVReader.readLine(in, stringQuote, valueSeparator)) != null) {
if (values.isEmpty()) {
continue;
}
importAd(adImport, lineNumber, existingAdCategoryMap, importedAdCategoryMap, customFieldMap, localSettings, headers, values);
lineNumber++;
cacheCleaner.clearCache();
}
} catch (final IOException e) {
throw new ValidationException("adImport.errorReading");
} finally {
IOUtils.closeQuietly(in);
}
return adImport;
}
@Override
public AdImport load(final Long id, final Relationship... fetch) throws EntityNotFoundException {
return adImportDao.load(id, fetch);
}
@Override
public void processImport(AdImport adImport) {
adImport = fetchService.fetch(adImport, AdImport.Relationships.CURRENCY);
final Map<ImportedAdCategory, AdCategory> importedCategories = new HashMap<ImportedAdCategory, AdCategory>();
// Iterate through each ad
final ImportedAdQuery adQuery = new ImportedAdQuery();
adQuery.fetch(ImportedAd.Relationships.EXISTING_CATEGORY, ImportedAd.Relationships.IMPORTED_CATEGORY, ImportedAd.Relationships.CUSTOM_VALUES);
adQuery.setAdImport(adImport);
adQuery.setStatus(ImportedAdQuery.Status.SUCCESS);
int count = 0;
final List<ImportedAd> importedAds = importedAdDao.search(adQuery);
for (final ImportedAd importedAd : importedAds) {
processAd(adImport, importedAd, importedCategories);
if (count % 20 == 0) {
// Every few records, clear the cache to avoid too many objects in memory
fetchService.clearCache();
}
count++;
}
// Delete the import after processing it
adImportDao.delete(adImport.getId());
}
@Override
public void purgeOld(Calendar time) {
// Only purge after 1 day of idleness
time = new TimePeriod(1, TimePeriod.Field.DAYS).remove(time);
for (final AdImport adImport : adImportDao.listBefore(time)) {
adImportDao.delete(adImport.getId());
}
}
@Override
public List<ImportedAd> searchImportedAds(final ImportedAdQuery params) {
return importedAdDao.search(params);
}
public void setAdCategoryServiceLocal(final AdCategoryServiceLocal adCategoryService) {
this.adCategoryService = adCategoryService;
}
public void setAdCustomFieldServiceLocal(final AdCustomFieldServiceLocal adCustomFieldService) {
this.adCustomFieldService = adCustomFieldService;
}
public void setAdImportDao(final AdImportDAO adImportDao) {
this.adImportDao = adImportDao;
}
public void setAdServiceLocal(final AdServiceLocal adService) {
this.adService = adService;
}
public void setElementServiceLocal(final ElementServiceLocal elementService) {
this.elementService = elementService;
}
public void setFetchServiceLocal(final FetchServiceLocal fetchService) {
this.fetchService = fetchService;
}
public void setImportedAdCategoryDao(final ImportedAdCategoryDAO importedAdCategoryDao) {
this.importedAdCategoryDao = importedAdCategoryDao;
}
public void setImportedAdDao(final ImportedAdDAO importedAdDao) {
this.importedAdDao = importedAdDao;
}
public void setSettingsServiceLocal(final SettingsServiceLocal settingsService) {
this.settingsService = settingsService;
}
@Override
public void validate(final AdImport AdImport) throws ValidationException {
getValidator().validate(AdImport);
}
/**
* Recursively add the category and it's children to the map, keyed by the full name
*/
private void appendCategory(final AdCategory adCategory, final Map<String, AdCategory> existingAdCategoryMap) {
// Just being cautious to avoid infinite loops in bad behaved databases
if (existingAdCategoryMap.values().contains(adCategory)) {
return;
}
existingAdCategoryMap.put(adCategory.getFullName(), adCategory);
for (final AdCategory child : adCategory.getChildren()) {
appendCategory(child, existingAdCategoryMap);
}
}
private Validator getValidator() {
final Validator validator = new Validator();
validator.property("currency").required();
return validator;
}
private Object handleCategory(final ImportedAd ad, final String value, final Map<String, AdCategory> existingAdCategoryMap, final Map<String, ImportedAdCategory> importedAdCategoryMap) {
if (StringUtils.isEmpty(value)) {
return null;
}
final String[] parts = StringUtils.split(value, ':');
Object category = null;
String fullPath = null;
// Validate the max level
if (parts.length > AdCategory.MAX_LEVEL) {
ad.setStatus(ImportedAd.Status.TOO_MANY_CATEGORY_LEVELS);
return null;
}
for (String part : parts) {
part = StringUtils.trimToNull(part);
if (part == null) {
ad.setStatus(ImportedAd.Status.INVALID_CATEGORY);
return null;
}
// Calculate the canonical full path (uses colon and a space as separators, as returned by AdCategory.getFullPath())
if (fullPath == null) {
fullPath = part;
} else {
fullPath += ": " + part;
}
// Check whether the category exists
final AdCategory existingCategory = existingAdCategoryMap.get(fullPath);
if (existingCategory != null) {
// There is an existing category
category = existingCategory;
} else {
ImportedAdCategory importedCategory = importedAdCategoryMap.get(fullPath);
if (importedCategory == null) {
// No existing category: create a new imported one
importedCategory = new ImportedAdCategory();
importedCategory.setAdImport(ad.getImport());
importedCategory.setName(part);
if (category instanceof AdCategory) {
importedCategory.setExistingParent((AdCategory) category);
} else if (category instanceof ImportedAdCategory) {
importedCategory.setImportedParent((ImportedAdCategory) category);
}
importedCategory = importedAdCategoryDao.insert(importedCategory);
importedAdCategoryMap.put(fullPath, importedCategory);
}
category = importedCategory;
}
}
return category;
}
private void importAd(final AdImport adImport, final int lineNumber, final Map<String, AdCategory> existingAdCategoryMap, final Map<String, ImportedAdCategory> importedAdCategoryMap, final Map<String, CustomField> customFieldMap, final LocalSettings localSettings, final List<String> headers, final List<String> values) {
final Map<String, String> customFieldValues = new HashMap<String, String>();
final CalendarConverter dateConverter = localSettings.getRawDateConverter();
final NumberConverter<BigDecimal> numberConverter = localSettings.getNumberConverter();
// Insert the ad
ImportedAd ad = new ImportedAd();
ad.setLineNumber(lineNumber);
ad.setImport(adImport);
ad.setStatus(ImportedAd.Status.SUCCESS);
ad = importedAdDao.insert(ad);
ad.setPublicationPeriod(new Period());
ad.setExternalPublication(true);
try {
ad.setCustomValues(new ArrayList<ImportedAdCustomFieldValue>());
// Process each field. Field names are lowercased to ignore case
for (int i = 0; i < headers.size() && i < values.size(); i++) {
final String field = StringUtils.trimToEmpty(headers.get(i)).toLowerCase();
final String value = StringUtils.trimToNull(values.get(i));
final boolean valueIsTrue = "true".equalsIgnoreCase(value) || "1".equals(value);
if ("owner".equals(field)) {
if (value != null) {
try {
final MemberUser user = (MemberUser) elementService.loadUser(value, RelationshipHelper.nested(User.Relationships.ELEMENT, Element.Relationships.GROUP));
ad.setOwner(user.getMember());
} catch (final Exception e) {
ad.setStatus(ImportedAd.Status.INVALID_OWNER);
ad.setErrorArgument1(value);
}
}
} else if ("title".equals(field)) {
ad.setTitle(value);
} else if ("description".equals(field)) {
ad.setDescription(value);
} else if ("html".equals(field)) {
ad.setHtml(valueIsTrue);
} else if ("publicationstart".equals(field)) {
try {
ad.getPublicationPeriod().setBegin(dateConverter.valueOf(value));
} catch (final Exception e) {
ad.setStatus(ImportedAd.Status.INVALID_PUBLICATION_START);
ad.setErrorArgument1(value);
break;
}
} else if ("publicationend".equals(field)) {
try {
ad.getPublicationPeriod().setEnd(dateConverter.valueOf(value));
} catch (final Exception e) {
ad.setStatus(ImportedAd.Status.INVALID_PUBLICATION_END);
ad.setErrorArgument1(value);
break;
}
} else if ("tradetype".equals(field)) {
// Only search is handled now, as it's the exception. Later, if it's null, offer is implied
if ("search".equalsIgnoreCase(value)) {
ad.setTradeType(TradeType.SEARCH);
}
} else if ("external".equals(field)) {
ad.setExternalPublication(valueIsTrue);
} else if ("price".equals(field)) {
try {
ad.setPrice(numberConverter.valueOf(value));
if (BigDecimal.ZERO.equals(ad.getPrice())) {
ad.setPrice(null);
}
} catch (final Exception e) {
ad.setStatus(ImportedAd.Status.INVALID_PRICE);
ad.setErrorArgument1(value);
break;
}
} else if ("category".equals(field)) {
final Object category = handleCategory(ad, value, existingAdCategoryMap, importedAdCategoryMap);
if (category instanceof AdCategory) {
ad.setExistingCategory((AdCategory) category);
} else if (category instanceof ImportedAdCategory) {
ad.setImportedCategory((ImportedAdCategory) category);
} else if (ad.getStatus() != null) {
// The handleCategory may have set the status. Set the argument and leave
ad.setErrorArgument1(value);
break;
}
} else if (customFieldMap.containsKey(field)) {
// Create a custom field value
final ImportedAdCustomFieldValue fieldValue = new ImportedAdCustomFieldValue();
fieldValue.setField(customFieldMap.get(field));
fieldValue.setValue(value);
ad.getCustomValues().add(fieldValue);
customFieldValues.put(field, value);
} else {
throw new UnknownColumnException(field);
}
}
// When there was an error, stop processing
if (ad.getStatus() != ImportedAd.Status.SUCCESS) {
return;
}
// Validate some data
if (ad.getOwner() == null) {
ad.setStatus(ImportedAd.Status.MISSING_OWNER);
return;
}
if (ad.getExistingCategory() == null && ad.getImportedCategory() == null) {
ad.setStatus(ImportedAd.Status.MISSING_CATEGORY);
return;
}
if (ad.getTitle() == null) {
ad.setStatus(ImportedAd.Status.MISSING_TITLE);
return;
}
if (ad.getDescription() == null) {
ad.setStatus(ImportedAd.Status.MISSING_DESCRIPTION);
return;
}
// Set some default data
final MemberGroupSettings groupSettings = ad.getOwner().getMemberGroup().getMemberSettings();
Calendar begin = ad.getPublicationPeriod().getBegin();
if (begin == null) {
// When there's no begin, assume today
begin = Calendar.getInstance();
ad.getPublicationPeriod().setBegin(begin);
}
final Calendar end = ad.getPublicationPeriod().getEnd();
if (end == null) {
// Without end, it's a permanent ad
// Check whether permanent ads are allowed
if (!groupSettings.isEnablePermanentAds()) {
ad.setStatus(ImportedAd.Status.MISSING_PUBLICATION_PERIOD);
return;
}
ad.setPermanent(true);
} else {
// Validate the publication period
if (begin.after(end)) {
ad.setStatus(ImportedAd.Status.PUBLICATION_BEGIN_AFTER_END);
return;
} else {
// Check the max publication time
final TimePeriod maxAdPublicationTime = groupSettings.getMaxAdPublicationTime();
if (!end.before(maxAdPublicationTime.add(begin))) {
ad.setStatus(ImportedAd.Status.MAX_PUBLICATION_EXCEEDED);
return;
}
}
}
if (ad.getTradeType() == null) {
ad.setTradeType(TradeType.OFFER);
}
switch (groupSettings.getExternalAdPublication()) {
case DISABLED:
ad.setExternalPublication(false);
break;
case ENABLED:
ad.setExternalPublication(true);
break;
}
// Save the custom field values
try {
adCustomFieldService.saveValues(ad);
} catch (final Exception e) {
ad.setStatus(ImportedAd.Status.INVALID_CUSTOM_FIELD);
if (e instanceof ValidationException) {
final ValidationException vex = (ValidationException) e;
final Map<String, Collection<ValidationError>> errorsByProperty = vex.getErrorsByProperty();
if (MapUtils.isNotEmpty(errorsByProperty)) {
final String fieldName = errorsByProperty.keySet().iterator().next();
ad.setErrorArgument1(fieldName);
final String fieldValue = StringUtils.trimToNull(customFieldValues.get(fieldName));
if (fieldValue == null) {
// When validation failed and the field is null, it's actually missing
ad.setStatus(ImportedAd.Status.MISSING_CUSTOM_FIELD);
} else {
ad.setErrorArgument2(fieldValue);
}
}
}
return;
}
} catch (final UnknownColumnException e) {
throw e;
} catch (final Exception e) {
ad.setStatus(ImportedAd.Status.UNKNOWN_ERROR);
ad.setErrorArgument1(e.toString());
} finally {
importedAdDao.update(ad);
}
}
private void processAd(final AdImport adImport, final ImportedAd importedAd, final Map<ImportedAdCategory, AdCategory> importedCategories) {
// Resolve the category first
AdCategory category = importedAd.getExistingCategory();
final ImportedAdCategory importedCategory = importedAd.getImportedCategory();
if (category == null && importedCategory != null) {
category = processCategory(importedCategory, importedCategories);
}
Ad ad = new Ad();
ad.setCategory(category);
// Without this fetch, Hibernate Search will bail, because the IsHasImages method is invoked
final Member owner = fetchService.fetch(importedAd.getOwner(), Member.Relationships.IMAGES, Member.Relationships.CUSTOM_VALUES);
if (owner != null) {
owner.setCustomValues(fetchService.fetch(owner.getCustomValues(), CustomFieldValue.Relationships.FIELD, CustomFieldValue.Relationships.POSSIBLE_VALUE));
ad.setOwner(owner);
}
ad.setTradeType(importedAd.getTradeType());
ad.setTitle(importedAd.getTitle());
ad.setDescription(importedAd.getDescription());
ad.setHtml(importedAd.isHtml());
ad.setPermanent(importedAd.isPermanent());
ad.setPublicationPeriod(importedAd.getPublicationPeriod());
ad.setExternalPublication(importedAd.isExternalPublication());
ad.setPrice(importedAd.getPrice());
if (ad.getPrice() != null) {
ad.setCurrency(adImport.getCurrency());
}
ad.setCustomValues(new ArrayList<AdCustomFieldValue>());
// Set the custom values
final Collection<ImportedAdCustomFieldValue> importedCustomValues = importedAd.getCustomValues();
if (importedCustomValues != null) {
for (final ImportedAdCustomFieldValue importedValue : importedCustomValues) {
final CustomField field = importedValue.getField();
final AdCustomFieldValue fieldValue = new AdCustomFieldValue();
fieldValue.setAd(ad);
fieldValue.setField(field);
if (field.getType() == CustomField.Type.ENUMERATED) {
fieldValue.setPossibleValue(importedValue.getPossibleValue());
} else if (field.getType() == CustomField.Type.MEMBER) {
fieldValue.setMemberValue(importedValue.getMemberValue());
} else {
fieldValue.setStringValue(importedValue.getStringValue());
}
ad.getCustomValues().add(fieldValue);
}
}
ad = adService.save(ad);
}
private AdCategory processCategory(ImportedAdCategory importedCategory, final Map<ImportedAdCategory, AdCategory> importedCategories) {
// Lookup in the map the already inserted category
importedCategory = fetchService.fetch(importedCategory, ImportedAdCategory.Relationships.EXISTING_PARENT, ImportedAdCategory.Relationships.IMPORTED_PARENT);
AdCategory category = importedCategories.get(importedCategory);
if (category == null) {
// Resolve the parent first
AdCategory existingParent = importedCategory.getExistingParent();
final ImportedAdCategory importedParent = importedCategory.getImportedParent();
if (existingParent == null && importedParent != null) {
existingParent = processCategory(importedParent, importedCategories);
}
// The first time this category is being used. Insert it
category = new AdCategory();
category.setParent(existingParent);
category.setActive(true);
category.setName(importedCategory.getName());
category = fetchService.fetch(adCategoryService.save(category));
importedCategories.put(importedCategory, category);
}
return category;
}
}