package controllers;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.avaje.ebean.Ebean;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import play.*;
import play.libs.Json;
import play.mvc.*;
import models.*;
import uk.bl.Const;
import uk.bl.api.Utils;
import uk.bl.exception.ActException;
import uk.bl.exception.TaxonomyNotFoundException;
import uk.bl.scope.Scope;
/**
* Currently using controller cookie auth rather than HTTP Basic via the @With(SecuredAction.class) technique.
*
* @author andy
*
*/
@Security.Authenticated(SecuredController.class)
public class APIController extends Controller {
//@With(SecuredAction.class)
public static Result viewAsJson(Long id) {
Target target = Target.findById(id);
if( target != null) {
return ok(Json.toJson(target));
} else {
return notFound("There is no Target with ID "+id);
}
}
/**
* @param id
* @return
*
* curl -v -H "Content-Type: application/json" -X PUT -d '{"uk_postal_address_url": "http://belly.co.uk"}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets/74
* curl -v -H "Content-Type: application/json" -X PUT -d '{"uk_postal_address_url": "http://belly.co.uk", "field_urls": ["http://www.poopoo.co.uk"]}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets/74
* curl -v -H "Content-Type: application/json" -X PUT -d '{"field_subjects": ["13","14"]}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets/74
* curl -v -H "Content-Type: application/json" -X PUT -d '{"field_collection_cats": ["2","4"]}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets/74
* curl -v -H "Content-Type: application/json" -X PUT -d '{"uk_postal_address_url": true,"field_uk_postal_address_url": "http://www.theguardian.com/"}' -u kinman.li@bl.uk:password http://www.webarchive.org.uk/actdev/api/targets/17356
* curl -v -H "Content-Type: application/json" -X PUT -d '{"uk_postal_address_url": "http://www.theguardian.com/","id": 17516,"field_uk_postal_address": true}' -u kinman.li@bl.uk:password http://www.webarchive.org.uk/actdev/api/targets/17516
* curl -v -H "Content-Type: application/json" -X PUT -d '{"uk_postal_address_url": "http://www.theguardian.com/","field_uk_postal_address": true}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets/1
*
* @throws ActException
*/
//@With(SecuredAction.class)
@BodyParser.Of(BodyParser.Json.class)
public static Result targetUpdate(Long id) throws ActException {
JsonNode node = request().body().asJson();
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.setSerializationInclusion(Include.NON_DEFAULT);
Logger.debug(node.asText());
try {
if (id == null) {
return badRequest("No ID found: " + id);
}
Logger.debug("UPDATE: " + node);
// process Targets here
Target dbTarget = Target.findById(id);
if (dbTarget == null) {
return badRequest("No Target Found for ID: " + id);
}
Logger.debug("ORIGINAL: " + dbTarget);
boolean dbWatched = dbTarget.isWatched();
WatchedTarget dbWatchedTarget = dbTarget.watchedTarget;
// find field and then update/save
ObjectReader updater = objectMapper.readerForUpdating(dbTarget);
Target merged = updater.readValue(node);
Logger.debug("MERGED: " + merged);
// url update
Target jsonTarget = objectMapper.readValue(node.toString(), Target.class);
if (jsonTarget.fieldUrls != null) {
if (merged.fieldUrls == null) {
merged.fieldUrls = new ArrayList<FieldUrl>();
}
if (jsonTarget.getField_urls() != null) {
for (String url : jsonTarget.getField_urls()) {
String trimmed = url.trim();
URL uri = new URI(trimmed).normalize().toURL();
String extFormUrl = uri.toExternalForm();
FieldUrl existingFieldUrl = FieldUrl.findByUrl(trimmed);
if (existingFieldUrl != null) {
Logger.debug("CONFLICT existingFieldUrl Url: " + existingFieldUrl.url);
return status(Http.Status.CONFLICT);
}
FieldUrl fieldUrl = new FieldUrl(extFormUrl.trim());
fieldUrl.domain = Scope.INSTANCE.getDomainFromUrl(fieldUrl.url);
merged.fieldUrls.add(fieldUrl);
}
}
}
List<String> fieldSubjects = jsonTarget.getField_subjects();
if (fieldSubjects != null) {
if (merged.subjects == null) {
merged.subjects = new ArrayList<Subject>();
}
for (String fieldSubject : fieldSubjects) {
try {
Subject subject = getSubject(fieldSubject);
if (subject.parent != null) {
merged.subjects = Utils.INSTANCE.processParentsSubjects(merged.subjects, subject.parent.id);
}
if (!merged.subjects.contains(subject)) {
merged.subjects.add(subject);
}
} catch(TaxonomyNotFoundException e) {
return badRequest("No Subject Found for : " + e);
} catch(Exception e) {
return badRequest("Issue with Subject: " + fieldSubject);
}
}
}
List<String> fieldCollections = jsonTarget.getField_collection_cats();
if (fieldCollections != null) {
if (merged.collections == null) {
merged.collections = new ArrayList<Collection>();
}
for (String fieldCollection : fieldCollections) {
try {
Collection collection = getCollection(fieldCollection);
Logger.debug("collection: " + collection);
if (collection.parent != null) {
merged.collections = Utils.INSTANCE.processParentsCollections(merged.collections, collection.parent.id);
}
if (!merged.collections.contains(collection)) {
merged.collections.add(collection);
}
} catch(TaxonomyNotFoundException e) {
return badRequest("No Collection Found for : " + fieldCollection);
} catch(Exception e) {
return badRequest("Issue with Collection: " + fieldCollection);
}
}
}
Logger.debug("fieldOrganisation...");
String fieldOrganisation = jsonTarget.getField_nominating_org();
if (StringUtils.isNotEmpty(fieldOrganisation)) {
Long orgId = Long.valueOf(fieldOrganisation);
Organisation organisation = Organisation.findById(orgId);
if (organisation == null) {
return badRequest("No Organisation Found for : " + orgId);
}
merged.organisation = organisation;
}
// "field_crawl_start_date": "1417255200"
if (jsonTarget.getField_crawl_start_date() != null) {
merged.crawlStartDate = Utils.INSTANCE.getDateFromSeconds(jsonTarget.getField_crawl_start_date());
}
Logger.debug("Dealing with field_crawl_end_date: "+ jsonTarget.getField_crawl_end_date());
if (jsonTarget.getField_crawl_end_date() != null) {
if( jsonTarget.getField_crawl_end_date() == 0 ) {
merged.crawlEndDate = null;
} else {
merged.crawlEndDate = Utils.INSTANCE.getDateFromSeconds(jsonTarget.getField_crawl_end_date());
}
}
if (StringUtils.isNotBlank(jsonTarget.getSelector())) {
Long selectorId = Long.valueOf(jsonTarget.getSelector());
User selector = User.findById(selectorId);
if (selector != null) {
merged.authorUser = selector;
}
}
// check any crawl permission
for( CrawlPermission cp: merged.crawlPermissions ) {
Logger.debug("GOT: "+cp);
Logger.debug("Checking: "+cp.contactPerson);
cp.contactPerson.save();
}
// Update as Watched if needed
Logger.info(" Watched is "+dbWatched+"/"+dbTarget.isWatched()+" - > "+ jsonTarget.isWatched());
Logger.info(" Watched is "+dbTarget.watchedTarget+" - > "+ jsonTarget.watchedTarget);
Logger.info(" Watched is "+merged);
if( !dbWatched && jsonTarget.isWatched() ) {
merged.watchedTarget.target = merged;
Ebean.save(merged.watchedTarget);
}
// Or remove/update:
if( dbWatched ) {
if( ! jsonTarget.isWatched() ) {
if( dbWatchedTarget.documents != null )
Ebean.delete(dbWatchedTarget.documents);
if( dbWatchedTarget.journalTitles != null)
Ebean.delete(dbWatchedTarget.journalTitles);
Ebean.delete(dbWatchedTarget);
merged.watchedTarget = null;
} else {
Ebean.update(merged.watchedTarget);
}
}
merged.runChecks();
merged.update();
String url = Play.application().configuration().getString("server_name") + Play.application().configuration().getString("application.context") + "/targets/" + merged.id;
Logger.debug("location: " + url);
response().setHeader(LOCATION, url);
Logger.debug("response 200 updated");
return ok(response().getHeaders().get(LOCATION));
} catch (Exception e) {
Logger.error("error: " + e);
return Results.internalServerError(e.getMessage());
}
}
/***
*
* NB Switched to cookie auth.
*
* curl -v -H "Content-Type: application/json" -X POST -d '{"title": "Turok 2","field_subjects": ["13","14"],"field_crawl_frequency": "monthly","field_nominating_org": "1","field_urls": ["http://turok99.com"],"field_collection_cats": ["8","9"],"field_crawl_start_date": "1417255200"}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets
* curl -v -H "Content-Type: application/json" -X POST -d '{"field_collection_cats": ["188"],"field_crawl_frequency": "daily","field_urls": ["http://www.independent.co.uk/news/uk/politics/"],"field_nominating_org": "1","title": "Independent, The: UK Politics"}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets
* curl -v -H "Content-Type: application/json" -X POST -d '{"field_collection_cats": ["188"],"field_crawl_frequency": "daily","field_urls": ["http://www.independent.co.uk/news/uk/politics/"],"field_nominating_org": "1"}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets
* curl -v -H "Content-Type: application/json" -X POST -d '{"field_urls": ["http://www.sdfsdfsf.co.ukxxxxxxxxxxxxxxxxxxx"],"title": "Daily Mail: Labours 500k help from tax avoidance firm","field_nominating_org": "1","field_collection_cats": ["188"],"field_crawl_frequency": "annual","field_crawl_end_date": "1425877200","field_crawl_start_date": "1425790800","field_subjects": [],"field_uk_postal_address": true,"uk_postal_address_url": "http://territest.co.uk", "field_via_correspondence": false,"field_professional_judgement": true,"field_professional_judgement_exp": ""}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets
* curl -v -H "Content-Type: application/json" -X POST -d '{"field_collection_cats": ["188"],"field_crawl_frequency": "daily","field_nominating_org": "1"}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets
* curl -v -H "Content-Type: application/json" -X POST -d '{
"field_subjects":[
"189"
],
"field_crawl_frequency":"annual",
"field_urls":[
"http://www.nottheguardian.com/commentisfree/2015/feb/10/tories-britain-democracy-auction-party-funds-donors-test"
],
"field_crawl_end_date":"1425877200",
"title":"Guardian, The: The Tories are putting Britain\u0027s democracy up for auction",
"uk_postal_address_url":"http://www.theguardian.com/",
"field_collection_cats":[
"188"
],
"field_nominating_org":"4",
"field_crawl_start_date":"1425790800",
"field_uk_postal_address":true}' -u kinman.li@bl.uk:password http://localhost:9000/actdev/api/targets
* @throws ActException
**/
//@With(SecuredAction.class)
@BodyParser.Of(BodyParser.Json.class)
public static Result bulkImport() throws ActException {
JsonNode node = request().body().asJson();
try {
if(node == null) {
return badRequest("Expecting Json data");
} else {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.setSerializationInclusion(Include.NON_DEFAULT);
Logger.debug("Parsing JSON: " + node);
// process Targets here
Target target = objectMapper.readValue(node.toString(), Target.class);
// Logger.debug("target: " + target);
// {
// "title": "Your Thurrock",
// "field_subjects": ["13","14"],
// "field_crawl_frequency": "monthly",
// "field_nominating_org": "1",
// "field_url": ["http://yourthurrock.com"],
// "field_collection_cats": ["10","11"],
// "field_crawl_start_date": "1417255200"
// }
// {
// "field_urls": [
// "http://www.dailymail.co.uk/news/article-2943512/Labour-s-500k-help-tax-avoidance-firm-Party-urged-stop-taking-advice-company-accused-controversial-schemes.html"
// ],
// "title": "Daily Mail: Labour's 500k help from 'tax avoidance' firm",
// "field_nominating_org": "1",
// "field_collection_cats": ["188"],
// "field_crawl_frequency": "annual",
// "field_crawl_end_date": "1425877200",
// "field_crawl_start_date": "1425790800",
// "field_subjects": [],
// "field_uk_postal_address": false,
// "field_via_correspondence": false,
// "field_professional_judgement": false,
// "field_professional_judgement_exp": ""
// }
if (StringUtils.isEmpty(target.title)) {
return badRequest("No Title Found for Target");
}
List<FieldUrl> fieldUrls = new ArrayList<FieldUrl>();
if (target.getField_urls() == null || target.getField_urls().isEmpty()) {
return badRequest("No URL(s) Found for Target");
}
for (String url : target.getField_urls()) {
String trimmed = url.trim();
URL uri = new URI(trimmed).normalize().toURL();
String extFormUrl = uri.toExternalForm();
FieldUrl existingFieldUrl = FieldUrl.hasDuplicate(trimmed);
if (existingFieldUrl != null) {
String duplicateUrl = Play.application().configuration().getString("server_name") + Play.application().configuration().getString("application.context") + "/targets/" + existingFieldUrl.target.id;
Logger.debug("CONFLICT existingFieldUrl Url: " + existingFieldUrl.url);
return status(Http.Status.CONFLICT,"Seed URL already associated with a current Target <a href=\"" + duplicateUrl + "\">" + duplicateUrl + "</a>");
}
FieldUrl fieldUrl = new FieldUrl(extFormUrl.trim());
fieldUrl.domain = Scope.INSTANCE.getDomainFromUrl(fieldUrl.url);
fieldUrls.add(fieldUrl);
Logger.debug("fieldUrls: " + fieldUrl.url);
}
// "field_url":[{"url":"http://www.childrenslegalcentre.com/"}],
if (!fieldUrls.isEmpty()) {
target.fieldUrls = fieldUrls;
}
Logger.debug("subjects...");
List<Subject> newSubjects = new ArrayList<Subject>();
List<String> fieldSubjects = target.getField_subjects();
if (fieldSubjects != null) {
for (String fieldSubject : fieldSubjects) {
try {
Subject subject = getSubject(fieldSubject);
if (subject.parent != null) {
newSubjects = Utils.INSTANCE.processParentsSubjects(newSubjects, subject.parent.id);
}
if (!newSubjects.contains(subject)) {
newSubjects.add(subject);
}
} catch(TaxonomyNotFoundException e) {
return badRequest("No Subject Found for : " + e);
} catch(Exception e) {
return badRequest("Issue with Subject: " + fieldSubject);
}
}
target.subjects = newSubjects;
}
if (Boolean.FALSE.equals(target.ukPostalAddress)) {
target.ukPostalAddressUrl = null;
}
Logger.debug("ukPostalAddress: " + target.ukPostalAddress);
Logger.debug("ukPostalAddressUrl: " + target.ukPostalAddressUrl);
// "field_crawl_frequency": "monthly"
Logger.debug("crawlFrequency...");
if (target.crawlFrequency != null) {
target.crawlFrequency = target.crawlFrequency.toUpperCase();
}
Logger.debug("fieldOrganisation...");
String fieldOrganisation = target.getField_nominating_org();
if (StringUtils.isNotEmpty(fieldOrganisation)) {
Long id = Long.valueOf(fieldOrganisation);
Organisation organisation = Organisation.findById(id);
if (organisation == null) {
return badRequest("No Organisation Found for : " + id);
}
target.organisation = organisation;
}
List<Collection> newCollections = new ArrayList<Collection>();
List<String> fieldCollections = target.getField_collection_cats();
if (fieldCollections != null) {
for (String fieldCollection : fieldCollections) {
try {
Collection collection = getCollection(fieldCollection);
if (collection.parent != null) {
newCollections = Utils.INSTANCE.processParentsCollections(newCollections, collection.parent.id);
}
if (!newCollections.contains(collection)) {
newCollections.add(collection);
}
} catch(TaxonomyNotFoundException e) {
return badRequest("No Collection Found for : " + fieldCollection);
} catch(Exception e) {
return badRequest("Issue with Collection: " + fieldCollection);
}
}
target.collections = newCollections;
}
Logger.debug("fieldSubjects: " + fieldSubjects);
Logger.debug("fieldOrganisations: " + fieldOrganisation);
Logger.debug("fieldCategories: " + fieldCollections);
// "field_crawl_start_date": "1417255200"
if (target.getField_crawl_start_date() != null) {
target.crawlStartDate = Utils.INSTANCE.getDateFromSeconds(target.getField_crawl_start_date());
}
if (target.getField_crawl_end_date() != null) {
target.crawlEndDate = Utils.INSTANCE.getDateFromSeconds(target.getField_crawl_end_date());
}
if (StringUtils.isNotBlank(target.getSelector())) {
Long selectorId = Long.valueOf(target.getSelector());
User selector = User.findById(selectorId);
if (selector != null) {
target.authorUser = selector;
}
}
target.url = "act-" + Utils.INSTANCE.createId();
target.runChecks();
// target.edit_url = Utils.INSTANCE.getWctUrl(target.vid);
// target.createdAt = Utils.INSTANCE.getDateFromSeconds(target.getCreated());
target.revision = Const.INITIAL_REVISION;
target.active = true;
target.selectionType = Const.SelectionType.SELECTION.name();
if (target.noLdCriteriaMet == null) {
target.noLdCriteriaMet = Boolean.FALSE;
}
if (target.keySite == null) {
target.keySite = Boolean.FALSE;
}
if (target.ignoreRobotsTxt == null) {
target.ignoreRobotsTxt = Boolean.FALSE;
}
// check any crawl permission
for( CrawlPermission cp: target.crawlPermissions ) {
Logger.debug("GOT: "+cp);
Logger.debug("Checking: "+cp.contactPerson);
cp.contactPerson.save();
}
target.save();
Logger.debug("target: " + target);
String url = Play.application().configuration().getString("server_name") + Play.application().configuration().getString("application.context") + "/targets/" + target.id;
Logger.debug("location: " + url);
response().setHeader(LOCATION, url);
target.save();
Logger.debug("response 201 created");
return created(response().getHeaders().get(LOCATION));
}
} catch (IllegalArgumentException e) {
return badRequest("URL invalid: " + e);
} catch (Exception e) {
Logger.error("error: " + e, e);
return Results.internalServerError(e.getMessage());
}
}
private static Collection getCollection(String stringId) throws IOException, TaxonomyNotFoundException, NumberFormatException {
Long id = Long.valueOf(stringId);
Collection collection = Collection.findById(id);
if (collection == null) {
throw new TaxonomyNotFoundException("No Collection id for: " + id);
}
return collection;
}
private static Subject getSubject(String stringId) throws IOException, TaxonomyNotFoundException {
Long id = Long.valueOf(stringId);
Subject subject = Subject.findById(id);
if (subject == null) {
throw new TaxonomyNotFoundException("No Subject id for: " + id);
}
return subject;
}
}