package controllers; import static play.data.Form.form; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TimeZone; import com.avaje.ebean.*; import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.Sheet; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.WorkbookFactory; import org.apache.commons.validator.routines.UrlValidator; import models.Collection; import models.CrawlPermission; import models.FieldUrl; import models.Flag; import models.License; import models.Organisation; import models.QaIssue; import models.Subject; import models.Tag; import models.Target; import models.Taxonomy; import models.User; import play.Logger; import play.Play; import play.data.DynamicForm; import play.data.Form; import play.data.validation.ValidationError; import play.libs.Json; import play.mvc.BodyParser; import play.mvc.Http.MultipartFormData; import play.mvc.Http.MultipartFormData.FilePart; import play.mvc.Result; import play.mvc.Security; import play.mvc.With; import uk.bl.Const; import uk.bl.Const.CrawlFrequency; import uk.bl.api.Utils; import uk.bl.api.models.CrawlFeedItem; import uk.bl.exception.ActException; import uk.bl.exception.WhoisException; import uk.bl.scope.Scope; import views.html.collections.sites; import views.html.licence.ukwalicenceresult; import views.html.infomessage; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; import com.rabbitmq.client.AMQP.BasicProperties; import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.client.Connection; import com.rabbitmq.client.Channel; import views.html.targets.blank; import views.html.targets.edit; import views.html.targets.list; import views.html.targets.lookup; import views.html.targets.view; import views.html.users.usersites; /** * Describe W3ACT project. */ @Security.Authenticated(SecuredController.class) public class TargetController extends AbstractController { final static Form<Target> targetForm = new Form<Target>(Target.class); private static final String DEFAULT_SORT_BY = "title"; private static final String DEFAULT_ORDER = "asc"; /** * Display the targets. */ public static Result index() { return GO_HOME; } /** * Display the paginated list of targets. * * @param page Current page number (starts from 0) * @param sortBy Column to be sorted * @param order Sort order (either asc or desc) * @param filter Filter applied on target urls */ public static Result lookup(int pageNo, String sortBy, String order, String filter) { Logger.debug("TargetController.lookup()"); String url = filter; if(url.startsWith("http://")) { url = url.replace("http://", ""); } if(url.startsWith("https://")) { url = url.replace("https://", ""); } // Re-enable www-ignoring part of search for lookup: if(url.startsWith("www.")) { url = url.replace("www.", ""); } if(url.endsWith("/")) { url = url.substring(0, url.length() - 1); } Logger.debug("After processing Filter::" + url); Query<FieldUrl> query = FieldUrl.find.fetch("target").fetch("target.organisation").where() .add(Expr.or(Expr.icontains("url", url), Expr.icontains("target.title", url))).query(); // Set up the sorting: if("title".equals(sortBy)) { query = query.orderBy("target.title" + " " + order); } else { if("organisation".equals(sortBy)) { query = query.orderBy("target.organisation.id" + " " + order); } else { if("seeds".equals(sortBy)) { query = query.orderBy("url" + " " + order); } else { if("frequency".equals(sortBy)) { query = query.orderBy("target.crawlFrequency" + " " + order); } else { if("active".equals(sortBy)) { query = query.orderBy("target.active" + " " + order); } } } } } // Finish the query: Page<FieldUrl> pages = query .findPagingList(20) .setFetchAhead(false).getPage(pageNo); Logger.debug("Total: " + pages.getTotalRowCount()); // Also check for clear match: Target matchTarget = null; try { FieldUrl isExistingFieldUrl = FieldUrl.hasDuplicate(filter.trim()); if(isExistingFieldUrl != null) { matchTarget = isExistingFieldUrl.target; } } catch(Exception e) { Logger.error("Problem looking up duplicate URLs.", e); } return ok( lookup.render( "Lookup " + filter, matchTarget, User.findByEmail(request().username()), filter, pages, sortBy, order) ); } /** * Display the paginated list of targets. * * @param page Current page number (starts from 0) * @param sortBy Column to be sorted * @param order Sort order (either asc or desc) * @param filter Filter applied on target urls * @param curatorId Author of the target * @param organisation The author's organisation * @param subject Target subject * @param crawlFrequency The crawl frequency * @param depth The crawl depth * @param collection The associated collection * @param license The license name * @param pageSize The number of Target entries on the page * @param flag The flag assigned by user */ public static Result list(int pageNo, String sortBy, String order, String filter, Long curatorId, Long organisationId, String subject, String crawlFrequencyName, String depthName, String collection, Long licenseId, int pageSize, Long flagId) { String url = filter; if(url.startsWith("http://")) { url = url.replace("http://", ""); } if(url.startsWith("https://")) { url = url.replace("https://", ""); } // The list page does NOT match so easily, www prefix NOT ignored: //if(url.startsWith("www.")) { // url = url.replace("www.", ""); //} if(url.endsWith("/")) { url = url.substring(0, url.length() - 1); } if(StringUtils.isBlank(sortBy)) { sortBy = DEFAULT_SORT_BY; order = DEFAULT_ORDER; } Logger.debug("After processing Filter::" + url); Logger.debug("Pre Targets.list(): " + pageNo + " - " + url + " - " + curatorId + " - " + organisationId + " - " + subject + " - " + crawlFrequencyName + " - " + depthName + " - " + collection + " - " + licenseId + " - " + pageSize + " - " + flagId); Page<Target> pageTargets = Target.pageTargets(pageNo, pageSize, sortBy, order, url, curatorId, organisationId, subject, crawlFrequencyName, depthName, collection, licenseId, flagId); User user = User.findByEmail(request().username()); List<License> licenses = License.findAllLicenses(); List<Long> subjectIds = new ArrayList<Long>(); String[] subjects = subject.split(", "); for(String sId : subjects) { if(StringUtils.isNotEmpty(sId)) { Long subjectId = Long.valueOf(sId); subjectIds.add(subjectId); } } JsonNode subjectData = getSubjectsDataByIds(subjectIds); List<Long> collectionIds = new ArrayList<Long>(); String[] collections = collection.split(", "); for(String cId : collections) { if(StringUtils.isNotEmpty(cId)) { Long collectionId = Long.valueOf(cId); collectionIds.add(collectionId); } } JsonNode collectionData = getCollectionsDataByIds(collectionIds); List<User> users = User.findAllSorted(); List<Organisation> organisations = Organisation.findAllSorted(); CrawlFrequency[] crawlFrequencies = Const.CrawlFrequency.values(); List<Flag> flags = Flag.findAllFlags(); Logger.debug("getTotalRowCount: " + pageTargets.getTotalRowCount()); return ok(list.render( "Targets", user, filter, pageTargets, sortBy, order, curatorId, organisationId, subject, crawlFrequencyName, depthName, collection, licenseId, pageSize, flagId, licenses, collectionData, subjectData, users, organisations, crawlFrequencies, flags) ); } public static String getTitle(Long id) { Target target = Target.findById(id); if(target != null) { return target.title; } else { return "Blank Target"; } } public static Result view(Long id) { Target target = Target.findById(id); if(target != null) { if(request().accepts("text/html")) { User user = User.findByEmail(request().username()); return ok(view.render(target, user)); } else { return ok(Json.toJson(target)); } } else { return notFound("There is no Target with ID " + id); } } public static Result viewAct(String url) { Target target = Target.findByUrl(url); User user = User.findByEmail(request().username()); return ok(view.render(target, user)); } public static Result viewWct(String url) { Target target = Target.findByWct(url); User user = User.findByEmail(request().username()); return ok(view.render(target, user)); } // @BodyParser.Of(BodyParser.Json.class) public static Result filterByJson(String url) { if(url.startsWith("http://")) { url = url.replace("http://", ""); } if(url.startsWith("https://")) { url = url.replace("https://", ""); } // Leave this out for now, making AJAX matches more precise: //if(url.startsWith("www.")) { // url = url.replace("www.", ""); //} if(url.endsWith("/")) { url = url.substring(0, url.length() - 1); } Logger.debug("search url is: " + url); JsonNode jsonData = null; if(url != null && url.length() >= 3) { Query<FieldUrl> query = FieldUrl.find.fetch("target").fetch("target.organisation").where() .add(Expr.or(Expr.icontains("url", url), Expr.icontains("target.title", url))).orderBy().asc("url"); // Only grab the first few... Page<FieldUrl> pages = query .findPagingList(20) .setFetchAhead(false).getPage(0); List<Target> targets = new ArrayList<Target>(); for(FieldUrl f : pages.getList()) { targets.add(f.target); } jsonData = Json.toJson(targets); } if(jsonData != null) { return ok(jsonData); } else { return notFound(); } } /** * This method enables searching for given URL and redirection in order to add new entry * if required. * * @return */ public static Result searchTargets() { DynamicForm requestData = Form.form().bindFromRequest(); if(requestData.get("pageSize") == null || (requestData.get("pageSize") != null && !Utils.INSTANCE.isNumeric(requestData.get("pageSize")))) { flash("message", "You may only enter a numeric page size."); return GO_HOME; } String action = requestData.get("action"); String filter = requestData.get("filter"); // if (StringUtils.isBlank(query)) { // Logger.debug("Target name is empty. Please write name in search window."); // flash("message", "Please enter a name in the search window"); // return GO_HOME; // } int pageNo = Integer.parseInt(requestData.get("p")); String sort = requestData.get("s"); String order = requestData.get("o"); int pageSize = Integer.parseInt(requestData.get("pageSize")); Long curatorId = Long.parseLong(requestData.get("curator")); Long organisationId = Long.parseLong(requestData.get("organisation")); Long licenseId = Long.parseLong(requestData.get("license")); String depthName = requestData.get("depth"); String crawlFrequencyName = requestData.get("crawlFrequency"); Long flagId = Long.parseLong(requestData.get("flag")); String subjectSelect = requestData.get("subjectSelect").replace("\"", ""); String collectionSelect = requestData.get("collectionSelect").replace("\"", ""); Logger.debug(filter + " " + pageNo + " " + sort + " " + order + " " + pageSize + " " + curatorId + " " + organisationId + " " + licenseId + " " + depthName + " " + crawlFrequencyName + " " + flagId + " " + collectionSelect + " " + subjectSelect); if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { if(action.equals("add")) { return redirect( routes.TargetController.newForm(filter) ); } else { if(action.equals("clear")) { return GO_HOME; } else { if(action.equals("export")) { List<Target> exportTargets = new ArrayList<Target>(); // Page<Target> page = Target.pageTargets(0, pageSize, sort, order, query, curator, organisation, subject, crawlFrequency, depth, collection, license, flag); // TODO: no time to do it now but this needs redoing with straight SQL/JPA to get the count Page<Target> page = Target.pageTargets(pageNo, pageSize, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); int rowCount = page.getTotalRowCount(); // Page<Target> pageAll = Target.pageTargets(0, rowCount, sort, order, query, curator, organisation, subject, crawlFrequency, depth, collection, license, flag); Page<Target> pageAll = Target.pageTargets(pageNo, rowCount, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, flagId); exportTargets.addAll(pageAll.getList()); Logger.debug("export size: " + exportTargets.size()); String csvData = Utils.INSTANCE.export(exportTargets); // return redirect(routes.TargetController.list(pageNo, sort, order, query, curator, organisation, // subject, crawlFrequency, depth, collection, license, pageSize, flag)); response().setContentType("text/csv; charset=utf-8"); response().setHeader("Content-disposition", "attachment; filename=\"target-export.csv"); return ok(csvData); // return redirect(routes.TargetController.list(pageNo, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, pageSize, flagId)); } else { if(action.equals("search") || action.equals("apply")) { return redirect(routes.TargetController.list(pageNo, sort, order, filter, curatorId, organisationId, subjectSelect, crawlFrequencyName, depthName, collectionSelect, licenseId, pageSize, flagId)); } else { return badRequest("This action is not allowed"); } } } } } } /** * This method enables searching for given URL and redirection in order to add new entry * if required. * * @return */ public static Result search() { DynamicForm form = DynamicForm.form().bindFromRequest(); String action = form.get("action"); String query = form.get("filter"); int pageNo = Integer.parseInt(form.get("p")); String sort = form.get("s"); String order = form.get("o"); if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { // check url boolean isValidUrl = Utils.INSTANCE.validUrl(query); Logger.debug("valid? " + isValidUrl); if(!isValidUrl) { ValidationError ve = new ValidationError("formUrl", "Invalid URL"); form.reject(ve); flash("message", "The URL entered is not valid. Please check and correct it, and click Search again"); return redirect(routes.TargetController.lookup(pageNo, sort, order, query)); } UrlValidator urlValidator = new UrlValidator(); if(!urlValidator.isValid(query)) { ValidationError ve = new ValidationError("formUrl", "Invalid URL"); form.reject(ve); flash("message", "The URL entered is not valid. Please check and correct it, and click Search again"); return redirect(routes.TargetController.lookup(pageNo, sort, order, query)); } if(action.equals("add")) { return redirect( routes.TargetController.newForm(query) ); } else { if(action.equals("search")) { Logger.debug("searching " + pageNo + " " + sort + " " + order); return redirect(routes.TargetController.lookup(pageNo, sort, order, query)); } else { return badRequest("This action is not allowed"); } } } } /** * Display the paginated list of targets. * * @param page Current page number (starts from 0) * @param sortBy Column to be sorted * @param order Sort order (either asc or desc) * @param filter Filter applied on target urls * @param collection_url Collection where targets search occurs */ public static Result collectionTargets(int pageNo, int pageSize, String sortBy, String order, String filter, Long collectionId) { Logger.debug("Targets.collectionTargets()"); Collection collection = Collection.findById(collectionId); Logger.debug("Collection: " + collection); Page<Target> pages = Target.pageCollectionTargets(pageNo, pageSize, sortBy, order, filter, collection.id); return ok( sites.render( collection, User.findByEmail(request().username()), filter, pages, sortBy, order, pageSize) ); } /** * @return */ public static Result allTargetsIDsAsJson() { List<Target> targets = Target.findAllActive(); Logger.debug("all targets: " + targets.size()); List<Long> target_ids = new ArrayList<Long>(targets.size()); for(Target t : targets) { target_ids.add(t.id); } return ok(Json.toJson(target_ids)); } /** * @return */ public static Result allTargetsAsJson(int pageNo, int pageLength) { List<Target> targets = Target.findAllActive(); int offset = pageNo * pageLength; if( offset > targets.size()) { return notFound("There are only "+targets.size()+" targets!"); } List<Target> targets_page = new ArrayList<Target>(pageLength); for(int i = 0; i < pageLength; i++) { if( offset+i >= targets.size()) break; targets_page.add(targets.get(offset+i)); } return ok(Json.toJson(targets_page)); } /** * @param collectionId * @return */ public static Result collectionTargetsAsJson(Long collectionId) { Collection collection = Collection.findById(collectionId); if(collection != null) { List<Target> targets = Target.allCollectionTargets(collection.id); Logger.debug("collections targets: " + targets.size()); List<Long> target_ids = new ArrayList<Long>(targets.size()); for(Target t : targets) { target_ids.add(t.id); } return ok(Json.toJson(target_ids)); } else { return notFound("There is not collection with ID " + collectionId); } } /** * Display the paginated list of targets. * * @param page Current page number (starts from 0) * @param sortBy Column to be sorted * @param order Sort order (either asc or desc) * @param filter Filter applied on target urls * @param subject_url Subject where targets search occurs */ public static Result subjectTargets(int pageNo, String sortBy, String order, String filter, Long subjectId) { Logger.debug("Targets.subjectTargets()"); return ok( views.html.subjects.sites.render( Subject.findById(subjectId), User.findByEmail(request().username()), filter, Target.pageSubjectTargets(pageNo, 10, sortBy, order, filter, subjectId), sortBy, order) ); } /** * Display the paginated list of targets for given organisation. * * @param page Current page number (starts from 0) * @param sortBy Column to be sorted * @param order Sort order (either asc or desc) * @param filter Filter applied on target urls * @param collection_url Collection where targets search occurs */ public static Result organisationTargets(int pageNo, String sortBy, String order, String filter, Long organisationId) { Logger.debug("Targets.organisationTargets()"); User user = User.findByEmail(request().username()); Organisation organisation = Organisation.findById(organisationId); Page<Target> pageTargets = Target.pageOrganisationTargets(pageNo, 10, sortBy, order, filter, organisation.id); return ok( views.html.organisations.sites.render( organisation, user, filter, pageTargets, sortBy, order) ); } /** * This method enables searching for given URL and particular collection. * * @return */ public static Result searchTargetsByCollection() { DynamicForm form = DynamicForm.form().bindFromRequest(); String action = form.get("action"); String query = form.get("url"); int pageNo = Integer.parseInt(form.get(Const.PAGE_NO)); int pageSize = Integer.parseInt(form.get(Const.PAGE_SIZE)); String sort = form.get(Const.SORT_BY); String order = form.get(Const.ORDER); String collection_url = form.get(Const.COLLECTION_URL); Collection collection = Collection.findByUrl(collection_url); if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { if(Const.SEARCH.equals(action)) { Logger.debug("searching " + pageNo + " " + pageSize + " " + sort + " " + order); return redirect(routes.TargetController.collectionTargets(pageNo, pageSize, sort, order, query, collection.id)); } else { return badRequest("This action is not allowed"); } } } /** * This method enables searching for given URL and particular subject. * * @return */ public static Result searchTargetsBySubject() { DynamicForm form = DynamicForm.form().bindFromRequest(); String action = form.get("action"); String query = form.get("url"); int pageNo = Integer.parseInt(form.get(Const.PAGE_NO)); String sort = form.get(Const.SORT_BY); String order = form.get(Const.ORDER); String subject_url = form.get("subject_url"); Long subjectId = Long.valueOf(subject_url); if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { if(Const.SEARCH.equals(action)) { Logger.debug("searching " + pageNo + " " + sort + " " + order); return redirect(routes.TargetController.subjectTargets(pageNo, sort, order, query, subjectId)); } else { return badRequest("This action is not allowed"); } } } /** * This method enables searching for given URL and particular organisation. * * @return */ public static Result searchTargetsByOrganisation() { DynamicForm form = DynamicForm.form().bindFromRequest(); String action = form.get(Const.ACTION); String query = form.get(Const.URL); int pageNo = Integer.parseInt(form.get(Const.PAGE_NO)); String sort = form.get(Const.SORT_BY); String order = form.get(Const.ORDER); String organisation_id = form.get("organisation_id"); Long organisationId = Long.valueOf(organisation_id); if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { if(Const.SEARCH.equals(action)) { Logger.debug("searching " + pageNo + " " + sort + " " + order); return redirect(routes.TargetController.organisationTargets(pageNo, sort, order, query, organisationId)); } else { return badRequest("This action is not allowed"); } } } /** * Display the paginated list of targets. * * @param page Current page number (starts from 0) * @param sortBy Column to be sorted * @param order Sort order (either asc or desc) * @param filter Filter applied on target urls * @param user_url User for whom targets search occurs * @param fastSubjects Taxonomy of type subject * @param collection Taxonomy of type collection */ public static Result userTargets(int pageNo, String sortBy, String order, String filter, Long userId, Long subjectId, Long collectionId) { User curator = User.findById(userId); User user = User.findByEmail(request().username()); Page<Target> pageTargets = Target.pageUserTargets(pageNo, 10, sortBy, order, filter, userId, subjectId, collectionId); List<Subject> subjects = Subject.findAllSubjects(); List<Collection> collections = Collection.findAllCollections(); Logger.debug("Targets.collectionTargets() " + userId + ", " + subjectId + ", " + collectionId); return ok( usersites.render( curator, user, filter, pageTargets, sortBy, order, subjectId, collectionId, subjects, collections) ); } /** * This method enables searching for given URL and particular user. * * @return */ public static Result searchTargetsByUser() { DynamicForm requestData = DynamicForm.form().bindFromRequest(); String action = requestData.get("action"); String query = requestData.get("url"); String user_id = requestData.get("userId"); Long userId = Long.valueOf(user_id); int pageNo = Integer.parseInt(requestData.get("p")); String sort = requestData.get("s"); String order = requestData.get("o"); String subject = requestData.get("subjectId"); Long subjectId = null; if(StringUtils.isNotBlank(subject)) { subjectId = Long.valueOf(subject); } String collection = requestData.get("collectionId"); Long collectionId = null; if(StringUtils.isNotBlank(collection)) { collectionId = Long.valueOf(collection); } // if (StringUtils.isBlank(query)) { // Logger.debug("Target name is empty. Please write name in search window."); // flash("message", "Please enter a name in the search window"); // return redirect(routes.Targets.userTargets(pageNo, sort, order, query, user_url, subject, collection)); // } if(StringUtils.isEmpty(action)) { return badRequest("You must provide a valid action"); } else { if(action.equals("add")) { String title = requestData.get("filter"); return redirect(routes.TargetController.newForm(title)); } if(action.equals("search")) { Logger.debug("searching " + pageNo + " " + sort + " " + order + " " + userId + ", " + subjectId + ", " + collectionId); return redirect(routes.TargetController.userTargets(pageNo, sort, order, query, userId, subjectId, collectionId)); } else { return badRequest("This action is not allowed"); } } } public static Result GO_HOME = redirect(routes.TargetController.list(0, "title", "asc", "", 0, 0, "", "", "", "", 0, 10, 0)); public static Result newForm(String title) { User user = User.findByEmail(request().username()); Form<Target> filledForm = Form.form(Target.class); Target target = new Target(); target.setDefaultValues(); if(StringUtils.isNotBlank(title)) { try { new URL(title); target.formUrl = title; } catch(MalformedURLException e) { target.title = title; } } target.revision = Const.INITIAL_REVISION; target.active = Boolean.TRUE; target.authorUser = user; target.organisation = user.organisation; target.language = Const.TargetLanguage.EN.toString(); Logger.debug("add target with url: " + target.url); Logger.debug("target title: " + target.title); filledForm = filledForm.fill(target); JsonNode collectionData = getCollectionsData(); JsonNode subjectData = getSubjectsData(); Map<String, String> authors = User.options(); List<Tag> tags = Tag.findAllTags(); List<Flag> flags = Flag.findAllFlags(); Map<String, String> qaIssues = QaIssue.options(); Map<String, String> languages = Const.TargetLanguage.options(); Map<String, String> selectionTypes = Const.SelectionType.options(); Map<String, String> scopeTypes = Const.ScopeType.options(); Map<String, String> depthTypes = Const.DepthType.options(); List<License> licenses = License.findAllLicenses(); Map<String, String> licenseStatuses = License.LicenseStatus.options(); Map<String, String> crawlFrequencies = Const.CrawlFrequency.options(); Map<String, String> siteStatuses = Const.SiteStatus.options(); Map<String, String> organisations = Organisation.options(); return ok(edit.render(filledForm, user, null, collectionData, subjectData, authors, tags, flags, qaIssues, languages, selectionTypes, scopeTypes, depthTypes, licenses, licenseStatuses, crawlFrequencies, siteStatuses, organisations, null, null, null, null)); } /** * Display the target edit panel for this URL. * * @param url The target identifier URL */ public static Result edit(Long id) { Logger.debug("Targets.edit() id::::: " + id); Target target = Target.findById(id); if(target == null) { return notFound("There is no Target with ID " + id); } // Make sure scope checks are up to date: target.runChecks(); target.update(); target.formUrl = target.fieldUrl(); target.subjectSelect = target.subjectIdsAsString(); target.collectionSelect = target.collectionIdsAsString(); Form<Target> filledForm = targetForm.fill(target); if(target.watchedTarget != null) { filledForm.data().putAll(FastSubjects.getFormData(target.watchedTarget.fastSubjects)); } User user = User.findByEmail(request().username()); target.language = Const.TargetLanguage.EN.toString(); JsonNode collectionData = getCollectionsData(target.collections); JsonNode subjectData = getSubjectsData(target.subjects); Map<String, String> authors = User.options(); List<Tag> tags = Tag.findAllTags(); List<Tag> targetTags = target.tags; List<Flag> flags = Flag.findAllFlags(); List<Flag> targetFlags = target.flags; Map<String, String> qaIssues = QaIssue.options(); Map<String, String> languages = Const.TargetLanguage.options(); Map<String, String> selectionTypes = Const.SelectionType.options(); Map<String, String> scopeTypes = Const.ScopeType.options(); Map<String, String> depthTypes = Const.DepthType.options(); List<License> licenses = License.findAllLicenses(); List<License> targetLicenses = target.licenses; Map<String, String> licenseStatuses = License.LicenseStatus.options(); Map<String, String> crawlFrequencies = Const.CrawlFrequency.options(); Map<String, String> siteStatuses = Const.SiteStatus.options(); Map<String, String> organisations = Organisation.options(); target.fieldUrl = target.fieldUrl(); // Logger.debug("DATE::::::::::::::::::::::::::::: "+filledForm.get().crawlEndDate.toString()); // DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm"); // formatter.setTimeZone(TimeZone.getTimeZone("UTC")); // try { // Date date = formatter.parse(filledForm.get().crawlEndDate.toString()); // filledForm.get().crawlEndDate = date; // } catch (ParseException e) { // e.printStackTrace(); // } Logger.debug("collections: " + target.collections.size()); return ok(edit.render(filledForm, user, id, collectionData, subjectData, authors, tags, flags, qaIssues, languages, selectionTypes, scopeTypes, depthTypes, licenses, licenseStatuses, crawlFrequencies, siteStatuses, organisations, null, targetTags, targetFlags, targetLicenses)); } public static Result delete(Long id) { Target target = Target.findById(id); Form<Target> filledForm = Form.form(Target.class); filledForm = filledForm.fill(target); if(!target.isDeletable()) { ValidationError ve = new ValidationError("formUrl", "Unable to delete Target as it references Instance(s), License(s) and/or Collection(s)"); filledForm.reject(ve); return info(filledForm, id); } if(target.hasDocuments()) { ValidationError ve = new ValidationError("watched", "Watched Targets with existing crawled documents can not be deleted."); filledForm.reject(ve); return info(filledForm, id); } if(target.watchedTarget != null) { //Ebean.delete(target.watchedTarget.documents); Ebean.delete(target.watchedTarget.journalTitles); } target.delete(); return redirect(routes.TargetController.index()); } /** * This method shows selected revision of a Target by given ID. * * @param id * @return */ public static Result viewrevision(Long id) { Target target = Target.findById(id); User user = User.findByEmail(request().username()); return ok(view.render(target, user)); } /** * Lists IDs for all targets of a given frequency, no matter what the status/schedule: * * @param frequency * @return */ @BodyParser.Of(BodyParser.Json.class) public static Result idsForFrequencyJson(String frequency) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.getByFrequency(frequency); List<Long> targetIds = new ArrayList<Long>(); for(Target t : targets) { targetIds.add(t.id); } jsonData = Json.toJson(targetIds); } return ok(jsonData); } /** * This method provides data exports for each possible crawl-frequency. * For each frequency this contains a list of Targets and associated * crawl metadata. * * @param frequency The crawl frequency e.g. 'daily' * @return list of Target objects */ @BodyParser.Of(BodyParser.Json.class) public static Result exportByFrequencyJson(String frequency) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportByFrequency(frequency); jsonData = Json.toJson(targets); } return ok(jsonData); } /** * As above, but only lists IDs * * @param frequency * @return */ @BodyParser.Of(BodyParser.Json.class) public static Result idsToCrawlByFrequencyJson(String frequency) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportByFrequency(frequency); List<Long> targetIds = new ArrayList<Long>(); for(Target t : targets) { targetIds.add(t.id); } jsonData = Json.toJson(targetIds); } return ok(jsonData); } /** * @param frequency * @return */ @BodyParser.Of(BodyParser.Json.class) public static Result crawlFeedByFrequencyJson(String frequency) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportByFrequency(frequency); List<CrawlFeedItem> targetIds = new ArrayList<CrawlFeedItem>(); for(Target t : targets) { targetIds.add(new CrawlFeedItem(t)); } jsonData = Json.toJson(targetIds); } return ok(jsonData); } /** * @param frequency * @return */ @BodyParser.Of(BodyParser.Json.class) public static Result crawlFeedOAFrequencyJson(String frequency) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportOAFrequency(frequency); List<CrawlFeedItem> targetIds = new ArrayList<CrawlFeedItem>(); for(Target t : targets) { targetIds.add(new CrawlFeedItem(t)); } jsonData = Json.toJson(targetIds); } return ok(jsonData); } /** * This method provides data exports for each possible crawl-frequency that are in legal deposit. * For each frequency this contains a list of Targets and associated * crawl metadata. * * @param frequency The crawl frequency e.g. 'daily' * @return list of Target objects * @throws ActException */ @BodyParser.Of(BodyParser.Json.class) public static Result exportLdFrequencyJson(String frequency) throws ActException { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportLdFrequency(frequency); jsonData = Json.toJson(targets); } return ok(jsonData); } /** * As above, but only lists IDs. * * @param frequency * @return * @throws ActException */ @BodyParser.Of(BodyParser.Json.class) public static Result idsToCrawlLdFrequencyJson(String frequency) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportLdFrequency(frequency); List<Long> targetIds = new ArrayList<Long>(); for(Target t : targets) { targetIds.add(t.id); } jsonData = Json.toJson(targetIds); } return ok(jsonData); } /** * @param frequency * @param whether to generate the paywall feed or not - i.e. this can exclude paywalled Targets, or be composed only of paywalled Targets. * @return * @throws ActException */ @BodyParser.Of(BodyParser.Json.class) public static Result crawlFeedLdFrequencyJson(String frequency, boolean generatePaywallFeed) { JsonNode jsonData = null; if(frequency != null) { List<Target> targets = new ArrayList<Target>(); targets = Target.exportLdFrequency(frequency); List<CrawlFeedItem> targetIds = new ArrayList<CrawlFeedItem>(); for(Target t : targets) { // Only add if(!generatePaywallFeed && t.secretId == null) { targetIds.add(new CrawlFeedItem(t)); } else if( generatePaywallFeed && t.secretId != null) { targetIds.add(new CrawlFeedItem(t)); } } jsonData = Json.toJson(targetIds); } return ok(jsonData); } /** * Example form with validation * * @return blank form for data entry */ public static Result blank() { Logger.debug("blank()"); return ok(blank.render(targetForm, User.findByEmail(request().username()))); } public static Result saveBlank() { Form<Target> filledForm = targetForm.bindFromRequest(); if(filledForm.hasErrors()) { Logger.debug("hasErrors: " + filledForm.hasErrors()); for(String key : filledForm.errors().keySet()) { Logger.debug("" + key); } return badRequest(blank.render(filledForm, User.findByEmail(request().username()))); } else { flash("success", "You've saved"); Logger.debug("saved"); return ok(blank.render(filledForm, User.findByEmail(request().username()))); } } /** * This method indicates to the user in a target record if data has been entered * by other users relating to NPLD status in another target record at a higher * level in the domain. * This is to avoid duplication of effort: users should not need to spend time * (outside ACT) doing the necessary research to populate the 'UK Postal Address', * 'Via Correspondence', and 'Professional Judgment' fields for abc.co.uk/directory * if those fields are already populated in a target record for abc.co.uk * @param fieldUrl The target URL * @return result as a flag. Result is true if: * (i) one or more of the three fields named above is not null in any other * target record at a higher level within the same domain AND * (ii) where both 'UK hosting' and 'UK top-level domain' = No. */ // public static boolean indicateNpldStatus(String fieldUrl) { // boolean res = false; // if (Target.getNpldStatusList(fieldUrl).size() > 0) { // res = true; // } // Logger.debug("indicateNpldStatus() res: " + res); // return res; // } /** * This method should give a list of the Target Titles and URLs for the * first three examples, in descending order of date of creation of the record. * * @param fieldUrl The target URL * @return result as a string */ public static String showNpldStatusList(String fieldUrl) { String res = ""; // try { // StringBuilder sb = new StringBuilder(); // List<Target> targets = Target.getNpldStatusList(fieldUrl); // Iterator<Target> itr = targets.iterator(); // while (itr.hasNext()) { // Target target = itr.next(); // sb.append(target.title + " " + target.fieldUrl()); // sb.append(System.getProperty("line.separator")); // } // res = sb.toString(); // } catch (Exception e) { // Logger.error("showNpldStatusList() " + e.getMessage()); // } return res; } /** * This method prepares Target form for sending info message * about errors * * @return edit page with form and info message */ public static Result info(Form<Target> form, Long id) { User user = User.findByEmail(request().username()); Map<String, String> authors = User.options(); List<Tag> tags = Tag.findAllTags(); List<Flag> flags = Flag.findAllFlags(); Map<String, String> qaIssues = QaIssue.options(); Map<String, String> languages = Const.TargetLanguage.options(); Map<String, String> selectionTypes = Const.SelectionType.options(); Map<String, String> scopeTypes = Const.ScopeType.options(); Map<String, String> depthTypes = Const.DepthType.options(); List<License> licenses = License.findAllLicenses(); Map<String, String> licenseStatuses = License.LicenseStatus.options(); Map<String, String> crawlFrequencies = Const.CrawlFrequency.options(); Map<String, String> siteStatuses = Const.SiteStatus.options(); Map<String, String> organisations = Organisation.options(); DynamicForm requestData = Form.form().bindFromRequest(); String tabStatus = requestData.get("tabstatus"); JsonNode collectionData = null; JsonNode subjectData = null; List<Tag> targetTags = null; List<Flag> targetFlags = null; List<License> targetLicenses = null; if(!form.hasErrors()) { Target target = form.get(); collectionData = getCollectionsData(target.collections); subjectData = getSubjectsData(target.subjects); targetTags = target.tags; targetFlags = target.flags; targetLicenses = target.licenses; } return badRequest(edit.render(form, user, id, collectionData, subjectData, authors, tags, flags, qaIssues, languages, selectionTypes, scopeTypes, depthTypes, licenses, licenseStatuses, crawlFrequencies, siteStatuses, organisations, tabStatus, targetTags, targetFlags, targetLicenses)); } private static String urlNoTrailingSlash(String url) { if(url.endsWith("/")) { url = url.substring(0, url.length() - 1); } return url; } /** * @param requestData * @param id * @return */ private static Result validateForm(DynamicForm requestData, Long id) throws ActException { Map<String, String[]> formParams = request().body().asFormUrlEncoded(); Form<Target> filledForm = form(Target.class).bindFromRequest(); User currentUser = User.findByEmail(request().username()); Target original = Target.findById(id); Logger.debug("hasGlobalErrors: " + filledForm.hasGlobalErrors()); if(filledForm.hasErrors()) { Logger.debug("hasErrors: " + filledForm.errors()); return info(filledForm, id); } String title = requestData.get("title"); if(title.trim().length() == 0) { flash("message", "Blank Title"); return info(filledForm, id); } String wctId = requestData.get("wctId"); if(StringUtils.isNotBlank(wctId) && !Utils.INSTANCE.isNumeric(wctId)) { flash("message", "Only numeric values are valid identifiers. Please check field 'WCT ID'."); return info(filledForm, id); } String sptId = requestData.get("sptId"); if(StringUtils.isNotBlank(sptId) && !Utils.INSTANCE.isNumeric(sptId)) { Logger.debug("Only numeric values are valid identifiers. Please check field 'SPT ID'."); flash("message", "Only numeric values are valid identifiers. Please check field 'SPT ID'."); return info(filledForm, id); } String legacySiteId = requestData.get("legacySiteId"); if(StringUtils.isNotBlank(legacySiteId) && !Utils.INSTANCE.isNumeric(legacySiteId)) { Logger.debug("Only numeric values are valid identifiers. Please check field 'LEGACY SITE ID'."); flash("message", "Only numeric values are valid identifiers. Please check field 'LEGACY SITE ID'."); return info(filledForm, id); } String author = requestData.get("authorUser.id"); if(StringUtils.isBlank(author)) { flash("message", "Please choose a Selector"); return info(filledForm, id); } List<License> newLicenses = new ArrayList<License>(); String[] licenseValues = formParams.get("licensesList"); // if it was originally set then let it go String openUkwa = requestData.get("openUkwaLicense"); if(licenseValues != null) { for(String licenseValue : licenseValues) { try { Long licenseId = Long.valueOf(licenseValue); License license = License.findById(licenseId); // could just use the ID instead if(StringUtils.isEmpty(openUkwa) && license.name.equals(Const.OPEN_UKWA_LICENSE)) { ValidationError ve = new ValidationError("licensesList", "It is not possible to attach an Open UKWA Licence directly to a target in this way. Please initiate the licensing process using the green button below"); filledForm.reject(ve); return info(filledForm, id); } newLicenses.add(license); } catch(NumberFormatException e) { Logger.debug("No license selected for " + filledForm.get().title); } } filledForm.get().licenses = newLicenses; } String webFormDate = requestData.get("webFormDateText"); if(StringUtils.isNotEmpty(webFormDate)) { DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); try { Date date = formatter.parse(webFormDate); filledForm.get().webFormDate = date; Logger.debug("webFormDate:::::::: " + date); } catch(ParseException e) { e.printStackTrace(); return info(filledForm, id); } } else { if(id != null) { Ebean.createUpdate(Target.class, "update target SET web_form_date=null where id=:id") .setParameter("id", id).execute(); } } String crawlFrequency = filledForm.get().crawlFrequency; if(StringUtils.isNotEmpty(crawlFrequency)) { String crawlStartDate = requestData.get("crawlStartDateText"); if(StringUtils.isNotEmpty(crawlStartDate)) { DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); try { Date date = formatter.parse(crawlStartDate); filledForm.get().crawlStartDate = date; Logger.debug("crawlStartDate:::::::: " + date); if(date.before(Calendar.getInstance().getTime())) { flash("warning", "<b>Warning! The crawl start date is in the past!</b><br/>This is normal for existing targets, but should not be the case for new targets."); } } catch(ParseException e) { e.printStackTrace(); return info(filledForm, id); } } else { if(id != null) { Ebean.createUpdate(Target.class, "update target SET crawl_start_date=null where id=:id") .setParameter("id", id).execute(); } } Logger.debug("crawl frequency: " + crawlFrequency); Logger.debug("crawlStartDate: " + crawlStartDate); if(!CrawlFrequency.DOMAINCRAWL.name().equals(crawlFrequency) && StringUtils.isEmpty(crawlStartDate)) { ValidationError ve = new ValidationError("crawlStartDateText", "Start Date is required when any crawl frequency other than 'Domain Crawl Only' is selected"); filledForm.reject(ve); return info(filledForm, id); } // Don't let non-archivists add or remove NEVERCRAWL status: if(original != null && CrawlFrequency.NEVERCRAWL.name().equals(original.crawlFrequency)) { // Do not remove: if((!CrawlFrequency.NEVERCRAWL.name().equals(crawlFrequency)) && !(currentUser.isArchivist() || currentUser.isSysAdmin()) ) { ValidationError ve = new ValidationError("crawlFrequency", "Only an archivist can change the crawl frequency from 'never crawl' to something else."); filledForm.reject(ve); return info(filledForm, id); } } else { // Do not add: if((CrawlFrequency.NEVERCRAWL.name().equals(crawlFrequency)) && !(currentUser.isArchivist() || currentUser.isSysAdmin()) ) { ValidationError ve = new ValidationError("crawlFrequency", "Only an archivist can change the crawl frequency to 'never crawl'."); filledForm.reject(ve); return info(filledForm, id); } } String crawlEndDate = requestData.get("crawlEndDateText"); if(StringUtils.isNotEmpty(crawlEndDate)) { DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy HH:mm"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); try { Date date = formatter.parse(crawlEndDate); filledForm.get().crawlEndDate = date; Logger.debug("crawlEndDate in date:::::::: " + date); } catch(ParseException e) { e.printStackTrace(); return info(filledForm, id); } } else { if(id != null) { Ebean.createUpdate(Target.class, "update target SET crawl_end_date=null where id=:id") .setParameter("id", id).execute(); } } } List<Tag> newTags = new ArrayList<Tag>(); String[] tagValues = formParams.get("tagsList"); if(original != null && tagValues != null) { for(String tagValue : tagValues) { Long tagId = Long.valueOf(tagValue); Tag tag = Tag.findTagById(tagId); newTags.add(tag); if(!original.tags.contains(tag)) { original.tags.add(tag); } } Ebean.update(original); filledForm.get().tags = newTags; } else { if(original != null && tagValues == null) { original.tags.clear(); Ebean.update(original); } } String[] flagValues = formParams.get("flagsList"); if(original != null && flagValues != null) { List<Flag> newFlags = new ArrayList<>(); for(String flagValue : flagValues) { Long flagId = Long.valueOf(flagValue); Flag flag = Flag.findById(flagId); newFlags.add(flag); if(!original.flags.contains(flag)) { original.flags.add(flag); } } for(Iterator<Flag> it = original.flags.iterator(); it.hasNext(); ) { if(!newFlags.contains(it.next())) { it.remove(); } } Ebean.update(original); filledForm.get().flags = null; } else { if(original != null && flagValues == null) { original.flags.clear(); Ebean.update(original); } } List<Subject> newSubjects = new ArrayList<Subject>(); String subjectSelect = requestData.get("subjectSelect").replace("\"", ""); Logger.debug("subjectSelect: " + subjectSelect); if(StringUtils.isNotEmpty(subjectSelect)) { String[] subjects = subjectSelect.split(", "); for(String sId : subjects) { Long subjectId = Long.valueOf(sId); Subject subject = Subject.findById(subjectId); if(subject.parent != null) { newSubjects = processParentsSubjects(newSubjects, subject.parent.id); } if(!newSubjects.contains(subject)) { newSubjects.add(subject); } } } filledForm.get().subjects = newSubjects; List<Collection> newCollections = new ArrayList<Collection>(); String collectionSelect = requestData.get("collectionSelect").replace("\"", ""); Logger.debug("collectionSelect: " + collectionSelect); if(StringUtils.isNotEmpty(collectionSelect)) { String[] collections = collectionSelect.split(", "); for(String cId : collections) { Long collectionId = Long.valueOf(cId); Collection collection = Collection.findById(collectionId); if(collection.parent != null) { newCollections = processParentsCollections(newCollections, collection.parent.id); } if(!newCollections.contains(collection)) { newCollections.add(collection); } } } filledForm.get().collections = newCollections; String dateOfPublication = requestData.get("dateOfPublicationText"); if(StringUtils.isNotEmpty(dateOfPublication)) { DateFormat formatter = new SimpleDateFormat("dd-MM-yyyy"); formatter.setTimeZone(TimeZone.getTimeZone("UTC")); try { Date date = formatter.parse(dateOfPublication); filledForm.get().dateOfPublication = date; } catch(ParseException e) { e.printStackTrace(); return info(filledForm, id); } } // Perform NPLD field consistency checks... // --- // UK Postal Address if(StringUtils.isEmpty(filledForm.get().ukPostalAddressUrl)) { if(Boolean.TRUE.equals(filledForm.get().ukPostalAddress)) { ValidationError ve = new ValidationError("ukPostalAddressUrl", "A URL indicating the UK Postal Address must be provided!"); filledForm.reject(ve); return info(filledForm, id); } else { filledForm.get().ukPostalAddress = false; } } else { // Auto-tick the checkbox filledForm.get().ukPostalAddress = true; } // via correspondence (which is apparently stored in the 'value' field!) if(StringUtils.isEmpty(filledForm.get().value)) { if(Boolean.TRUE.equals(filledForm.get().viaCorrespondence)) { ValidationError ve = new ValidationError("value", "A description of the correspondence must be provided!"); filledForm.reject(ve); return info(filledForm, id); } else { filledForm.get().viaCorrespondence = false; } } else { // Auto-tick the checkbox filledForm.get().viaCorrespondence = true; } // professional judgement if(StringUtils.isEmpty(filledForm.get().professionalJudgementExp)) { if(Boolean.TRUE.equals(filledForm.get().professionalJudgement)) { ValidationError ve = new ValidationError("professionalJudgementExp", "An outline of your professional judgement must be provided!"); filledForm.reject(ve); return info(filledForm, id); } else { filledForm.get().professionalJudgement = false; } } else { // Auto-tick the checkbox filledForm.get().professionalJudgement = true; } //Hidden flag if(filledForm.get().hidden == null) { filledForm.get().hidden = Boolean.FALSE; } //Key Site flag if(filledForm.get().keySite == null) { filledForm.get().keySite = Boolean.FALSE; } // Ensure items NOT edited herein are re-attached: if(original != null) { filledForm.get().crawlPermissions = original.crawlPermissions; Logger.warn("Attempting to repair " + original.crawlPermissions + "(" + original.crawlPermissions.size() + ")"); filledForm.get().instances = original.instances; filledForm.get().lookupEntries = original.lookupEntries; } // Run scoping checks: filledForm.get().runChecks(); // Check if those checks invalidate the noLDmet: if((Boolean.TRUE.equals(filledForm.get().isUkHosting) || Boolean.TRUE.equals(filledForm.get().isTopLevelDomain) || Boolean.TRUE.equals(filledForm.get().isUkRegistration) || Boolean.TRUE.equals(filledForm.get().ukPostalAddress) || Boolean.TRUE.equals(filledForm.get().viaCorrespondence) || Boolean.TRUE.equals(filledForm.get().professionalJudgement)) && Boolean.TRUE.equals(filledForm.get().noLdCriteriaMet) ) { ValidationError ve = new ValidationError("noLdCriteriaMet", "One of the checks for NPLD permission has been passed. Please unselect the 'No LD Criteria Met' field and click Save again"); filledForm.reject(ve); return info(filledForm, id); } Logger.debug("noLdCriteriaMet: " + filledForm.get().noLdCriteriaMet); if(filledForm.get().noLdCriteriaMet == null) { filledForm.get().noLdCriteriaMet = Boolean.FALSE; } //Updating licence status // ANJ: Note that manual changes at this top-level should not modify the existing CrawlPermissions. // Commenting out until the needs are more clearly defined. /* if (filledForm.get().getLatestCrawlPermission() != null) { filledForm.get().getLatestCrawlPermission().status = filledForm.get().licenseStatus; if( Const.CrawlPermissionStatus.GRANTED.getValue().equalsIgnoreCase(filledForm.get().getLatestCrawlPermission().status)){ filledForm.get().getLatestCrawlPermission().grantedAt = Utils.INSTANCE.getCurrentTimeStamp(); } if( Const.CrawlPermissionStatus.PENDING.getValue().equalsIgnoreCase(filledForm.get().getLatestCrawlPermission().status)){ filledForm.get().getLatestCrawlPermission().requestedAt = Utils.INSTANCE.getCurrentTimeStamp(); } filledForm.get().getLatestCrawlPermission().update(); } */ Result checkUrlsResult = validateUrls(requestData, id, filledForm); if(checkUrlsResult != null) { return checkUrlsResult; } filledForm.get().runChecks(); // Transaction start Ebean.beginTransaction(); try { // Lock the child field_url table to guard against a race condition that can result in duplicate URLs being created Ebean.createSqlUpdate("LOCK TABLE field_url;").execute(); Result checkUniqueUrlResult = validateCheckForExistingUrl(id, filledForm); if(checkUniqueUrlResult != null) { return checkUniqueUrlResult; } if(id != null) { filledForm.get().update(id); } else { filledForm.get().url = "act-" + Utils.INSTANCE.createId(); filledForm.get().active = Boolean.TRUE; filledForm.get().save(); } Ebean.commitTransaction(); } finally { Ebean.endTransaction(); } // Transaction end // The watchedTarget is not cascaded automatically, so we handle it here: String watchedString = getFormParam("watched"); boolean watched = false; if(watchedString != null) { watched = Boolean.parseBoolean(watchedString); } Logger.info("WATCHED status is " + watched + " from " + watchedString); if(original != null) { if(!watched && original.isWatched()) { Ebean.delete(original.watchedTarget.documents); Ebean.delete(original.watchedTarget.journalTitles); Ebean.delete(original.watchedTarget); } else { if(watched && !original.isWatched()) { filledForm.get().watchedTarget.target = original; filledForm.get().watchedTarget.fastSubjects = FastSubjects.getFastSubjects(filledForm); Ebean.save(filledForm.get().watchedTarget); } else { if(watched && original.isWatched()) { original.watchedTarget.documentUrlScheme = filledForm.get().watchedTarget.documentUrlScheme; original.watchedTarget.archivistNotesWT = filledForm.get().watchedTarget.archivistNotesWT; original.watchedTarget.fastSubjects = FastSubjects.getFastSubjects(filledForm); Ebean.update(original.watchedTarget); } } } } if(id != null) { flash("message", "Target " + filledForm.get().title + " has been updated"); } else { flash("success", "Target " + filledForm.get().title + " has been created"); } return redirect(routes.TargetController.view(filledForm.get().id)); } private static Result validateUrls(DynamicForm requestData, Long id, Form<Target> filledForm) throws ActException { String fieldUrl = requestData.get("formUrl"); Logger.debug("\n\nfieldUrl: " + fieldUrl); if(StringUtils.isNotEmpty(fieldUrl)) { String[] urls = fieldUrl.split(","); List<FieldUrl> fieldUrls = new ArrayList<FieldUrl>(); long position = 0; for(String url : urls) { String trimmed = url.trim(); Logger.debug("trimmed " + trimmed); URL uri; try { uri = new URI(trimmed).normalize().toURL(); } catch(MalformedURLException | URISyntaxException | IllegalArgumentException e) { ValidationError ve = new ValidationError("formUrl", "The URL entered is not valid. Please check and correct it, and click Save again"); filledForm.reject(ve); return info(filledForm, id); } UrlValidator urlValidator = new UrlValidator(); if(!urlValidator.isValid(trimmed)) { ValidationError ve = new ValidationError("formUrl", "The URL entered is not valid. Please check and correct it, and click Save again"); filledForm.reject(ve); return info(filledForm, id); } String extFormUrl = uri.toExternalForm(); FieldUrl fu = new FieldUrl(extFormUrl.trim()); boolean isValidUrl = Utils.INSTANCE.validUrl(trimmed); Logger.debug("valid? " + isValidUrl); if(!isValidUrl) { ValidationError ve = new ValidationError("formUrl", "The URL entered is not valid. Please check and correct it, and click Save again"); filledForm.reject(ve); flash("message", "Invalid URL."); return redirect(routes.TargetController.edit(id)); } Logger.debug("Adding url: " + trimmed + " at position " + position); fu.position = position; position++; Logger.debug("extFormUrl: " + extFormUrl); fieldUrls.add(fu); } filledForm.get().fieldUrls = fieldUrls; Logger.debug("fieldUrls: " + filledForm.get().fieldUrls); } return null; } private static Result validateCheckForExistingUrl(Long id, Form<Target> filledForm) throws ActException { List<FieldUrl> fieldUrls = filledForm.get().fieldUrls; for(FieldUrl fieldUrl : fieldUrls) { FieldUrl isExistingFieldUrl = FieldUrl.hasDuplicate(fieldUrl.url); Logger.debug("For url " + fieldUrl.url); Logger.debug("Found existing FieldUrl " + isExistingFieldUrl); Logger.debug("Got filledForm.get().id: " + filledForm.get().id); if(isExistingFieldUrl != null) { if(!isExistingFieldUrl.target.id.equals(id)) { Logger.debug("Found existing FieldUrl.target " + isExistingFieldUrl.target); String duplicateUrl = Play.application().configuration().getString("server_name") + Play.application().configuration().getString("application.context") + "/targets/" + isExistingFieldUrl.target.id; ValidationError ve = new ValidationError("formUrl", "Seed URL already associated with another Target <a href=\"" + duplicateUrl + "\">" + duplicateUrl + "</a>"); filledForm.reject(ve); return info(filledForm, id); } } } return null; } public static Result update(Long id) throws ActException { DynamicForm requestData = form().bindFromRequest(); String action = requestData.get("action"); if(StringUtils.isNotEmpty(action)) { if(action.equals("request")) { return redirect(routes.CrawlPermissionController.licenceRequestForTarget(id)); } else { if(action.equals("save")) { return validateForm(requestData, id); } else { if(action.equals("archive")) { return redirect(routes.TargetController.archive(id)); } else { if(action.equals("delete")) { return delete(id); } } } } } return null; } /** * This method saves changes on given target in a new target object * completed by revision comment. The "version" field in the Target object * contains the timestamp of the change and the last version is marked by * flag "active". Remaining Target objects with the same URL are not active. * * @return * @throws ActException */ public static Result save() throws ActException { DynamicForm requestData = form().bindFromRequest(); Map<String, String[]> formParams = request().body().asFormUrlEncoded(); Form<Target> filledForm = form(Target.class).bindFromRequest(); if(filledForm.hasErrors()) { Logger.debug("errors: " + filledForm.errors()); return info(filledForm, null); } return validateForm(requestData, null); } private static List<Subject> processParentsSubjects(List<Subject> subjects, Long parentId) { Subject parent = Subject.findById(parentId); if(!subjects.contains(parent)) { subjects.add(parent); } if(parent.parent != null) { processParentsSubjects(subjects, parent.parent.id); } return subjects; } private static List<Collection> processParentsCollections(List<Collection> collections, Long parentId) { Collection parent = Collection.findById(parentId); if(!collections.contains(parent)) { collections.add(parent); } if(parent.parent != null) { processParentsCollections(collections, parent.parent.id); } return collections; } /** * This method pushes a message onto a RabbitMQ queue for given target * using global settings from project configuration file. * * @param target The field URL of the target * @return */ public static Result archive(Long id) { Target target = Target.findById(id); Logger.debug("archiveTarget() " + target); if(target != null) { if(!target.isInScopeAllOrInheritedWithoutLicense()) { return ok(infomessage.render("On-demand archiving is only supported for NPLD targets.")); } // Send the message: try { String queueHost = Play.application().configuration().getString(Const.QUEUE_HOST); String queuePort = Play.application().configuration().getString(Const.QUEUE_PORT); String queueName = Play.application().configuration().getString(Const.QUEUE_NAME); String routingKey = Play.application().configuration().getString(Const.ROUTING_KEY); String exchangeName = Play.application().configuration().getString(Const.EXCHANGE_NAME); Logger.debug("archiveTarget() queue host: " + queueHost); Logger.debug("archiveTarget() queue port: " + queuePort); Logger.debug("archiveTarget() queue name: " + queueName); Logger.debug("archiveTarget() routing key: " + routingKey); Logger.debug("archiveTarget() exchange name: " + exchangeName); JsonNode jsonData = Json.toJson(target); String message = jsonData.toString(); Logger.debug("Crawl Now message: " + message); ConnectionFactory factory = new ConnectionFactory(); if(queueHost != null) { factory.setHost(queueHost); } if(queuePort != null) { factory.setPort(Integer.parseInt(queuePort)); } Connection connection = factory.newConnection(); Channel channel = connection.createChannel(); channel.exchangeDeclare(exchangeName, "direct", true); channel.queueDeclare(queueName, true, false, false, null); channel.queueBind(queueName, exchangeName, routingKey); BasicProperties.Builder propsBuilder = new BasicProperties.Builder(); propsBuilder.deliveryMode(2); channel.basicPublish(exchangeName, routingKey, propsBuilder.build(), message.getBytes()); channel.close(); connection.close(); } catch(IOException e) { String msg = "There was a problem queuing this crawl instruction. Please refer to the system administrator."; User currentUser = User.findByEmail(request().username()); StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); String msgFullTrace = sw.toString(); Logger.error(msgFullTrace); if(currentUser.hasRole("sys_admin")) { msg = msgFullTrace; } return ok(infomessage.render(msg)); } catch(Exception e) { String msg = "There was a problem queuing this crawl instruction. Please refer to the system administrator."; User currentUser = User.findByEmail(request().username()); StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); String msgFullTrace = sw.toString(); Logger.error(msgFullTrace); if(currentUser.hasRole("sys_admin")) { msg = msgFullTrace; } return ok(infomessage.render(msg)); } } else { Logger.debug("There was a problem sending the message. Target field for archiving is empty"); return ok(infomessage.render("There was a problem sending the message. Target field for archiving is empty")); } return ok( ukwalicenceresult.render() ); } /** * This method is checking scope for given URL and returns result in JSON format. * * @param url * @return JSON result * @throws WhoisException */ public static Result isInScope(String url) throws WhoisException { Logger.debug("isInScope controller: " + url); boolean res = Scope.INSTANCE.check(url, null, false); return ok(Json.toJson(res)); } /** * This method calculates collection children - objects that have parents. * * @param url The identifier for parent * @param targetUrl This is an identifier for current target object * @return child collection in JSON form */ public static String getChildren(String url, String targetUrl) { // Logger.debug("getChildren() target URL: " + targetUrl); String res = ""; final StringBuffer sb = new StringBuffer(); sb.append(", \"children\":"); List<Collection> childSuggestedCollections = Collection.getChildLevelCollections(url); if(childSuggestedCollections.size() > 0) { sb.append(getTreeElements(childSuggestedCollections, targetUrl, false)); res = sb.toString(); // Logger.debug("getChildren() res: " + res); } return res; } /** * Mark collections that are stored in target object as selected * * @param collectionUrl The collection identifier * @param targetUrl This is an identifier for current target object * @return */ public static String checkSelection(String collectionUrl, String targetUrl) { String res = ""; if(targetUrl != null && targetUrl.length() > 0) { Target target = Target.findByUrl(targetUrl); if(target.collections != null && target.collections.contains(collectionUrl)) { res = "\"select\": true ,"; } } return res; } /** * Mark preselected collections as selected in filter * * @param collectionUrl The collection identifier * @param targetUrl This is an identifier for current target object * @return */ public static String checkSelectionFilter(String collectionUrl, String targetUrl) { String res = ""; if(targetUrl != null && targetUrl.length() > 0) { if(collectionUrl != null && targetUrl.equals(collectionUrl)) { res = "\"select\": true ,"; } } return res; } /** * This method calculates first order collections for filtering. * * @param collectionList The list of all collections * @param targetUrl This is an identifier for current target object * @param parent This parameter is used to differentiate between root and children nodes * @return collection object in JSON form */ public static String getTreeElementsFilter(List<Collection> collectionList, String targetUrl, boolean parent) { // Logger.debug("getTreeElements() target URL: " + targetUrl); String res = ""; if(collectionList.size() > 0) { final StringBuffer sb = new StringBuffer(); sb.append("["); Iterator<Collection> itr = collectionList.iterator(); boolean firstTime = true; while(itr.hasNext()) { Collection collection = itr.next(); // Logger.debug("add collection: " + collection.name + ", with url: " + collection.url + // ", parent:" + collection.parent + ", parent size: " + collection.parent.length()); if((parent && collection.parent == null) || !parent || collection.parent == null) { if(firstTime) { firstTime = false; } else { sb.append(", "); } // Logger.debug("added"); sb.append("{\"title\": \"" + collection.name + "\"," + checkSelectionFilter(collection.url, targetUrl) + " \"key\": \"" + collection.url + "\"" + getChildren(collection.url, targetUrl) + "}"); } } // Logger.debug("collectionList level size: " + collectionList.size()); sb.append("]"); res = sb.toString(); // Logger.debug("getTreeElements() res: " + res); } return res; } // [{"title":"100 Best Sites (95)","url":"/actdev/collections/act-170","key":"\"act-170\""}, // {"title":"19th Century English Literature (0)","url":"/actdev/collections/act-153","key":"\"act-153\""}, // {"title":"Commonwealth Games Glasgow 2014 (625)","url":"/actdev/collections/act-252","key":"\"act-252\"","children":[ // {"title":"Competitors (350)","url":"/actdev/collections/act-266","key":"\"act-266\""},{"title":"Cultural Programme (44)","url":"/actdev/collections/act-295","key":"\"act-295\""},{"title":"Organisational bodies/venues (57)","url":"/actdev/collections/act-267","key":"\"act-267\""},{"title":"Press & Media Comment (41)","url":"/actdev/collections/act-269","key":"\"act-269\""},{"title":"Sponsors (15)","url":"/actdev/collections/act-270","key":"\"act-270\""},{"title":"Sports (405)","url":"/actdev/collections/act-268","key":"\"act-268\"","children":[{"title":"Aquatics (74)","url":"/actdev/collections/act-287","key":"\"act-287\""},{"title":"Athletics (107)","url":"/actdev/collections/act-286","key":"\"act-286\""},{"title":"Badminton (18)","url":"/actdev/collections/act-285","key":"\"act-285\""},{"title":"Boxing (26)","url":"/actdev/collections/act-284","key":"\"act-284\""},{"title":"Cycling (34)","url":"/actdev/collections/act-283","key":"\"act-283\""},{"title":"Gymnastics (29)","url":"/actdev/collections/act-282","key":"\"act-282\""},{"title":"Hockey (17)","url":"/actdev/collections/act-281","key":"\"act-281\""},{"title":"Judo (9)","url":"/actdev/collections/act-280","key":"\"act-280\""},{"title":"Lawn bowls (11)","url":"/actdev/collections/act-279","key":"\"act-279\""},{"title":"Netball (6)","url":"/actdev/collections/act-278","key":"\"act-278\""},{"title":"Rugby Sevens (16)","url":"/actdev/collections/act-277","key":"\"act-277\""},{"title":"Shooting (14)","url":"/actdev/collections/act-276","key":"\"act-276\""},{"title":"Squash (16)","url":"/actdev/collections/act-275","key":"\"act-275\""},{"title":"Table tennis (6)","url":"/actdev/collections/act-274","key":"\"act-274\""},{"title":"Triathlon (7)","url":"/actdev/collections/act-273","key":"\"act-273\""},{"title":"Weightlifting (7)","url":"/actdev/collections/act-272","key":"\"act-272\""},{"title":"Wrestling (7)","url":"/actdev/collections/act-271","key":"\"act-271\""}]}]},{"title":"Conservative Party Website deletions - Press articles November 2013 (10)","url":"/actdev/collections/act-175","key":"\"act-175\""},{"title":"Ebola (192)","url":"/actdev/collections/act-296","key":"\"act-296\""},{"title":"European Parliament Elections 2014 (1743)","url":"/actdev/collections/act-250","key":"\"act-250\"","children":[{"title":"Academia & think tanks (15)","url":"/actdev/collections/act-257","key":"\"act-257\""},{"title":"Blogs (215)","url":"/actdev/collections/act-263","key":"\"act-263\""},{"title":"Candidates (520)","url":"/actdev/collections/act-262","key":"\"act-262\""},{"title":"EU Institutions (1)","url":"/actdev/collections/act-260","key":"\"act-260\""},{"title":"Interest groups (63)","url":"/actdev/collections/act-254","key":"\"act-254\""},{"title":"Opinion Polls (7)","url":"/actdev/collections/act-261","key":"\"act-261\""},{"title":"Political Parties: National (99)","url":"/actdev/collections/act-256","key":"\"act-256\""},{"title":"Political Parties: Regional & Local (65)","url":"/actdev/collections/act-258","key":"\"act-258\""},{"title":"Press & Media Comment (744)","url":"/actdev/collections/act-255","key":"\"act-255\""},{"title":"Regulation and Guidance (13)","url":"/actdev/collections/act-253","key":"\"act-253\""},{"title":"Social Media (1)","url":"/actdev/collections/act-259","key":"\"act-259\""}]},{"title":"Evolving role of libraries in the UK (0)","url":"/actdev/collections/act-154","key":"\"act-154\""},{"title":"First World War Centenary, 2014-18 (208)","url":"/actdev/collections/act-174","key":"\"act-174\"","children":[{"title":"Heritage Lottery Fund (66)","url":"/actdev/collections/act-265","key":"\"act-265\""}]}, // {"title":"Health and Social Care Act 2012 - NHS Reforms (752)","url":"/actdev/collections/act-24","key":"\"act-24\"","children":[ // {"title":"NHS (720)","url":"/actdev/collections/act-25","key":"\"act-25\"","children":[ // {"title":"Acute Trusts (161)","url":"/actdev/collections/act-27","key":"\"act-27\""},{"title":"Ambulance Trusts (12)","url":"/actdev/collections/act-136","key":"\"act-136\""},{"title":"Campaigning and Advocacy Groups (18)","url":"/actdev/collections/act-148","key":"\"act-148\""},{"title":"Cancer Networks (28)","url":"/actdev/collections/act-137","key":"\"act-137\""},{"title":"Care Trust (29)","url":"/actdev/collections/act-139","key":"\"act-139\""},{"title":"Clinical Commissioning Groups (191)","url":"/actdev/collections/act-28","key":"\"act-28\""},{"title":"Gateways (11)","url":"/actdev/collections/act-141","key":"\"act-141\""},{"title":"Health and Wellbeing Boards (108)","url":"/actdev/collections/act-29","key":"\"act-29\""},{"title":"Healthwatch (127)","url":"/actdev/collections/act-144","key":"\"act-144\""},{"title":"Legislation (17)","url":"/actdev/collections/act-143","key":"\"act-143\""},{"title":"Local Authorities (142)","url":"/actdev/collections/act-166","key":"\"act-166\""},{"title":"Local Involvement Networks (LINks) (159)","url":"/actdev/collections/act-30","key":"\"act-30\""},{"title":"Mental Health Trusts (50)","url":"/actdev/collections/act-138","key":"\"act-138\""},{"title":"NHS programmes (3)","url":"/actdev/collections/act-142","key":"\"act-142\""},{"title":"Press Comment (248)","url":"/actdev/collections/act-150","key":"\"act-150\""},{"title":"Primary Care Trusts (15)","url":"/actdev/collections/act-26","key":"\"act-26\""},{"title":"Private and voluntary sector providers (8)","url":"/actdev/collections/act-169","key":"\"act-169\""},{"title":"Professional Bodies Trade Union (49)","url":"/actdev/collections/act-147","key":"\"act-147\""},{"title":"Public Health Agencies (1)","url":"/actdev/collections/act-134","key":"\"act-134\""},{"title":"Public Health England (138)","url":"/actdev/collections/act-149","key":"\"act-149\""},{"title":"Regulators & Central Government (1)","url":"/actdev/collections/act-135","key":"\"act-135\""},{"title":"Social Media (Facebook, Twitter etc) (42)","url":"/actdev/collections/act-167","key":"\"act-167\""},{"title":"Special Health Authorities (13)","url":"/actdev/collections/act-140","key":"\"act-140\""},{"title":"Specialised Commissioning Group (12)","url":"/actdev/collections/act-145","key":"\"act-145\""},{"title":"Strategic Health Authorities (20)","url":"/actdev/collections/act-82","key":"\"act-82\"","children":[{"title":"London SHA Cluster (1)","url":"/actdev/collections/act-83","key":"\"act-83\"","children":[{"title":"London SHA (1)","url":"/actdev/collections/act-87","key":"\"act-87\"","children":[{"title":"London (1)","url":"/actdev/collections/act-31","key":"\"act-31\""},{"title":"North Central London (6)","url":"/actdev/collections/act-33","key":"\"act-33\""},{"title":"North East London and City (9)","url":"/actdev/collections/act-32","key":"\"act-32\""},{"title":"North West London (8)","url":"/actdev/collections/act-34","key":"\"act-34\""},{"title":"South East London (6)","url":"/actdev/collections/act-36","key":"\"act-36\""},{"title":"South West London (6)","url":"/actdev/collections/act-35","key":"\"act-35\""}]}]},{"title":"Midlands and East SHA Cluster (4)","url":"/actdev/collections/act-85","key":"\"act-85\"","children":[{"title":"East Midlands (1)","url":"/actdev/collections/act-91","key":"\"act-91\"","children":[{"title":"Derby City and Derbyshire (2)","url":"/actdev/collections/act-53","key":"\"act-53\""},{"title":"East Midlands (2)","url":"/actdev/collections/act-52","key":"\"act-52\""},{"title":"Leicestershire County & Rutland and Leicestershire City (2)","url":"/actdev/collections/act-54","key":"\"act-54\""},{"title":"Lincolnshire (1)","url":"/actdev/collections/act-55","key":"\"act-55\""},{"title":"Milton Keynes and Northamptonshire (2)","url":"/actdev/collections/act-56","key":"\"act-56\""},{"title":"Nottinghamshhire County and Nottingham City (2)","url":"/actdev/collections/act-57","key":"\"act-57\""}]},{"title":"East of England (1)","url":"/actdev/collections/act-92","key":"\"act-92\"","children":[{"title":"Bedfordshire and Luton (1)","url":"/actdev/collections/act-59","key":"\"act-59\""},{"title":"Cambridgeshire and Peterborough (3)","url":"/actdev/collections/act-62","key":"\"act-62\""},{"title":"Hertfordshire (1)","url":"/actdev/collections/act-58","key":"\"act-58\""},{"title":"Norfolk and Waveney (2)","url":"/actdev/collections/act-63","key":"\"act-63\""},{"title":"North, Mid and East Essex (1)","url":"/actdev/collections/act-60","key":"\"act-60\""},{"title":"South Essex (2)","url":"/actdev/collections/act-61","key":"\"act-61\""},{"title":"Suffolk (1)","url":"/actdev/collections/act-64","key":"\"act-64\""}]},{"title":"West Midlands (1)","url":"/actdev/collections/act-93","key":"\"act-93\"","children":[{"title":"Arden (2)","url":"/actdev/collections/act-65","key":"\"act-65\""},{"title":"Birmingham and Solihull (5)","url":"/actdev/collections/act-66","key":"\"act-66\""},{"title":"Black Country (4)","url":"/actdev/collections/act-67","key":"\"act-67\""},{"title":"Staffordshire (3)","url":"/actdev/collections/act-69","key":"\"act-69\""},{"title":"West Mercia (5)","url":"/actdev/collections/act-70","key":"\"act-70\""},{"title":"West Midlands (1)","url":"/actdev/collections/act-68","key":"\"act-68\""}]}]},{"title":"North of England SHA Cluster (7)","url":"/actdev/collections/act-84","key":"\"act-84\"","children":[{"title":"North East (3)","url":"/actdev/collections/act-88","key":"\"act-88\"","children":[{"title":"County Durham and Darlington (3)","url":"/actdev/collections/act-37","key":"\"act-37\""},{"title":"North of Tyne (5)","url":"/actdev/collections/act-38","key":"\"act-38\""},{"title":"South of Tyne (4)","url":"/actdev/collections/act-39","key":"\"act-39\""},{"title":"Tees (5)","url":"/actdev/collections/act-40","key":"\"act-40\""}]},{"title":"North West (1)","url":"/actdev/collections/act-89","key":"\"act-89\"","children":[{"title":"Cheshire, Warrington, Wirral (4)","url":"/actdev/collections/act-41","key":"\"act-41\""},{"title":"Cumbria (1)","url":"/actdev/collections/act-42","key":"\"act-42\""},{"title":"Greater Manchester (9)","url":"/actdev/collections/act-43","key":"\"act-43\""},{"title":"Lancashire (5)","url":"/actdev/collections/act-44","key":"\"act-44\""},{"title":"Merseyside (3)","url":"/actdev/collections/act-45","key":"\"act-45\""}]},{"title":"Yorkshire and the Humber (3)","url":"/actdev/collections/act-90","key":"\"act-90\"","children":[{"title":"Bradford (2)","url":"/actdev/collections/act-49","key":"\"act-49\""},{"title":"Calderdale, Kirklees and Wakefield (3)","url":"/actdev/collections/act-46","key":"\"act-46\""},{"title":"Humber (5)","url":"/actdev/collections/act-47","key":"\"act-47\""},{"title":"Leeds (1)","url":"/actdev/collections/act-50","key":"\"act-50\""},{"title":"North Yorkshire and York (1)","url":"/actdev/collections/act-51","key":"\"act-51\""},{"title":"South Yorkshire and Bassetlaw (5)","url":"/actdev/collections/act-48","key":"\"act-48\""}]}]},{"title":"South of England SHA Cluster (1)","url":"/actdev/collections/act-86","key":"\"act-86\"","children":[{"title":"South Central (0)","url":"/actdev/collections/act-94","key":"\"act-94\"","children":[{"title":"Berkshire (6)","url":"/actdev/collections/act-71","key":"\"act-71\""},{"title":"Buckinghamshire and Oxfordshire (2)","url":"/actdev/collections/act-73","key":"\"act-73\""},{"title":"Southampton, Hampshire, Isle of Wight & Portsmouth (5)","url":"/actdev/collections/act-72","key":"\"act-72\""}]},{"title":"South East Coast (1)","url":"/actdev/collections/act-95","key":"\"act-95\"","children":[{"title":"Kent and Medway (4)","url":"/actdev/collections/act-74","key":"\"act-74\""},{"title":"Surrey (1)","url":"/actdev/collections/act-75","key":"\"act-75\""},{"title":"Sussex (5)","url":"/actdev/collections/act-76","key":"\"act-76\""}]},{"title":"South West (0)","url":"/actdev/collections/act-96","key":"\"act-96\"","children":[{"title":"Bath, North East Somerset and Wiltshire (2)","url":"/actdev/collections/act-77","key":"\"act-77\""},{"title":"Bournemouth, Poole and Dorset (2)","url":"/actdev/collections/act-78","key":"\"act-78\""},{"title":"Bristol, North Somerset and South Gloucestershire (4)","url":"/actdev/collections/act-79","key":"\"act-79\""},{"title":"Devon, Plymouth and Torbay (3)","url":"/actdev/collections/act-80","key":"\"act-80\""},{"title":"Gloucestershire and Swindon (3)","url":"/actdev/collections/act-81","key":"\"act-81\""}]}]}]},{"title":"Think Tanks (30)","url":"/actdev/collections/act-146","key":"\"act-146\""}]}]},{"title":"History of Libraries Collection (0)","url":"/actdev/collections/act-155","key":"\"act-155\""},{"title":"History of the Book (0)","url":"/actdev/collections/act-156","key":"\"act-156\""},{"title":"Legal Aid Reform (0)","url":"/actdev/collections/act-157","key":"\"act-157\""},{"title":"Margaret Thatcher (77)","url":"/actdev/collections/act-151","key":"\"act-151\""},{"title":"Nelson Mandela (174)","url":"/actdev/collections/act-178","key":"\"act-178\""},{"title":"News Sites (576)","url":"/actdev/collections/act-173","key":"\"act-173\"","children":[{"title":"Hyperlocal (515)","url":"/actdev/collections/act-297","key":"\"act-297\""}]},{"title":"Oral History in the UK (2)","url":"/actdev/collections/act-158","key":"\"act-158\""},{"title":"Political Action and Communication (5)","url":"/actdev/collections/act-159","key":"\"act-159\""},{"title":"Religion, politics and law since 2005 (9)","url":"/actdev/collections/act-160","key":"\"act-160\""},{"title":"Religion/Theology (237)","url":"/actdev/collections/act-172","key":"\"act-172\""},{"title":"Scottish Independence Referendum (1144)","url":"/actdev/collections/act-171","key":"\"act-171\"","children":[{"title":"Charities, Churches and Third Sector (16)","url":"/actdev/collections/act-293","key":"\"act-293\""},{"title":"Commercial Publishers (1)","url":"/actdev/collections/act-294","key":"\"act-294\""},{"title":"Government (UK and Scottish) (83)","url":"/actdev/collections/act-291","key":"\"act-291\""},{"title":"National Campaigning Groups (80)","url":"/actdev/collections/act-288","key":"\"act-288\""},{"title":"Political Parties and Trade Unions (249)","url":"/actdev/collections/act-289","key":"\"act-289\""},{"title":"Press, Media & Comment (263)","url":"/actdev/collections/act-292","key":"\"act-292\""},{"title":"Think Tanks and Research Institutes (47)","url":"/actdev/collections/act-290","key":"\"act-290\""}]},{"title":"Slavery and Abolition in the Caribbean (0)","url":"/actdev/collections/act-161","key":"\"act-161\""},{"title":"Tour de France (Yorkshire) 2014 (60)","url":"/actdev/collections/act-264","key":"\"act-264\""},{"title":"UK relations with the Low Countries (0)","url":"/actdev/collections/act-162","key":"\"act-162\""},{"title":"UK response to Philippines disaster 2013 (501)","url":"/actdev/collections/act-176","key":"\"act-176\""},{"title":"Video Games (0)","url":"/actdev/collections/act-163","key":"\"act-163\""},{"title":"Winter Olympics Sochi 2014 (128)","url":"/actdev/collections/act-177","key":"\"act-177\""}] // [{"title":"Health and Social Care Act 2012 - NHS Reforms","select":true,"key":"act-24","children":[ // {"title":"NHS","key":"act-25","children":[ // {"title":"Campaigning and Advocacy Groups","key":"act-148"},{"title":"Acute Trusts","key":"act-27"},{"title":"Clinical Commissioning Groups","key":"act-28"},{"title":"Strategic Health Authorities","key":"act-82","children":[ // {"title":"London SHA Cluster","key":"act-83","children":[{"title":"London SHA","key":"act-87","children":[ // {"title":"North West London","key":"act-34"},{"title":"London","key":"act-31"},{"title":"North East London and City","key":"act-32"},{"title":"North Central London","key":"act-33"},{"title":"South West London","key":"act-35"},{"title":"South East London","key":"act-36"}]}]},{"title":"Midlands and East SHA Cluster","key":"act-85","children":[{"title":"East Midlands","key":"act-91","children":[{"title":"Nottinghamshhire County and Nottingham City","key":"act-57"},{"title":"Derby City and Derbyshire","key":"act-53"},{"title":"Leicestershire County & Rutland and Leicestershire City","key":"act-54"},{"title":"Lincolnshire","key":"act-55"},{"title":"Milton Keynes and Northamptonshire","key":"act-56"}]},{"title":"West Midlands","key":"act-93","children":[{"title":"Black Country","key":"act-67"},{"title":"Arden","key":"act-65"},{"title":"Birmingham and Solihull","key":"act-66"},{"title":"Staffordshire","key":"act-69"},{"title":"West Mercia","key":"act-70"}]},{"title":"East of England","key":"act-92","children":[{"title":"Hertfordshire","key":"act-58"},{"title":"Bedfordshire and Luton","key":"act-59"},{"title":"North, Mid and East Essex","key":"act-60"},{"title":"South Essex","key":"act-61"},{"title":"Cambridgeshire and Peterborough","key":"act-62"},{"title":"Norfolk and Waveney","key":"act-63"},{"title":"Suffolk","key":"act-64"}]}]},{"title":"North of England SHA Cluster","key":"act-84","children":[{"title":"North West","key":"act-89","children":[{"title":"Greater Manchester","key":"act-43"},{"title":"Cheshire, Warrington, Wirral","key":"act-41"},{"title":"Cumbria","key":"act-42"},{"title":"Lancashire","key":"act-44"},{"title":"Merseyside","key":"act-45"}]},{"title":"North East","key":"act-88","children":[{"title":"County Durham and Darlington","key":"act-37"},{"title":"North of Tyne","key":"act-38"},{"title":"South of Tyne","key":"act-39"},{"title":"Tees","key":"act-40"}]},{"title":"Yorkshire and the Humber","key":"act-90","children":[{"title":"Calderdale, Kirklees and Wakefield","key":"act-46"},{"title":"Humber","key":"act-47"},{"title":"South Yorkshire and Bassetlaw","key":"act-48"},{"title":"Bradford","key":"act-49"},{"title":"Leeds","key":"act-50"},{"title":"North Yorkshire and York","key":"act-51"}]}]},{"title":"South of England SHA Cluster","key":"act-86","children":[{"title":"South Central","key":"act-94","children":[{"title":"Berkshire","key":"act-71"},{"title":"Southampton, Hampshire, Isle of Wight & Portsmouth","key":"act-72"},{"title":"Buckinghamshire and Oxfordshire","key":"act-73"}]},{"title":"South East Coast","key":"act-95","children":[{"title":"Kent and Medway","key":"act-74"},{"title":"Surrey","key":"act-75"},{"title":"Sussex","key":"act-76"}]},{"title":"South West","key":"act-96","children":[{"title":"Bath, North East Somerset and Wiltshire","key":"act-77"},{"title":"Bournemouth, Poole and Dorset","key":"act-78"},{"title":"Bristol, North Somerset and South Gloucestershire","key":"act-79"},{"title":"Devon, Plymouth and Torbay","key":"act-80"},{"title":"Gloucestershire and Swindon","key":"act-81"}]}]}]},{"title":"Health and Wellbeing Boards","key":"act-29"},{"title":"Local Involvement Networks (LINks)","key":"act-30"},{"title":"Care Trust","key":"act-139"},{"title":"Public Health England","key":"act-149"},{"title":"Special Health Authorities","key":"act-140"},{"title":"Public Health Agencies","key":"act-134"},{"title":"Ambulance Trusts","key":"act-136"},{"title":"Mental Health Trusts","key":"act-138"},{"title":"Gateways","key":"act-141"},{"title":"NHS programmes","key":"act-142"},{"title":"Professional Bodies Trade Union","key":"act-147"},{"title":"Legislation","key":"act-143"},{"title":"Healthwatch","key":"act-144"},{"title":"Specialised Commissioning Group","key":"act-145"},{"title":"Think Tanks","key":"act-146"},{"title":"Press Comment","key":"act-150"},{"title":"Cancer Networks","key":"act-137"},{"title":"Regulators & Central Government","key":"act-135"},{"title":"Local Authorities","key":"act-166"},{"title":"Social Media (Facebook, Twitter etc)","key":"act-167"},{"title":"Primary Care Trusts","key":"act-26"},{"title":"Private and voluntary sector providers","key":"act-169"}]}]},{"title":"Scottish Independence Referendum","key":"act-171","children":[{"title":"Press, Media & Comment","key":"act-292"},{"title":"Political Parties and Trade Unions","key":"act-289"},{"title":"Think Tanks and Research Institutes","key":"act-290"},{"title":"National Campaigning Groups","key":"act-288"},{"title":"Government (UK and Scottish)","key":"act-291"},{"title":"Charities, Churches and Third Sector","key":"act-293"}]},{"title":"Science, Technology & Medicine","key":"act-99","children":[{"title":"Physics","key":"act-108"},{"title":"Astronomy","key":"act-107"}]},{"title":"Religion, politics and law since 2005","key":"act-160"},{"title":"News Sites","key":"act-173"},{"title":"100 Best Sites","key":"act-170"},{"title":"Margaret Thatcher","key":"act-151"},{"title":"First World War Centenary, 2014-18","key":"act-174","children":[{"title":"Heritage Lottery Fund","key":"act-265"}]},{"title":"Religion/Theology","key":"act-172"},{"title":"European Parliament Elections 2014","key":"act-250","children":[{"title":"Political Parties: National","key":"act-256"},{"title":"Academia & think tanks","key":"act-257"},{"title":"Interest groups","key":"act-254"},{"title":"Regulation and Guidance","key":"act-253"},{"title":"Blogs","key":"act-263"},{"title":"Political Parties: Regional & Local","key":"act-258"},{"title":"Social Media","key":"act-259"},{"title":"EU Institutions","key":"act-260"},{"title":"Opinion Polls","key":"act-261"},{"title":"Candidates","key":"act-262"}]},{"title":"Nelson Mandela","key":"act-178"},{"title":"UK response to Philippines disaster 2013","key":"act-176"},{"title":"Commonwealth Games Glasgow 2014","key":"act-252","children":[{"title":"Organisational bodies/venues","key":"act-267"},{"title":"Sports","key":"act-268","children":[{"title":"Rugby Sevens","key":"act-277"},{"title":"Badminton","key":"act-285"},{"title":"Cycling","key":"act-283"},{"title":"Wrestling","key":"act-271"},{"title":"Hockey","key":"act-281"},{"title":"Boxing","key":"act-284"},{"title":"Aquatics","key":"act-287"},{"title":"Athletics","key":"act-286"},{"title":"Gymnastics","key":"act-282"},{"title":"Judo","key":"act-280"},{"title":"Lawn bowls","key":"act-279"},{"title":"Netball","key":"act-278"},{"title":"Shooting","key":"act-276"},{"title":"Squash","key":"act-275"},{"title":"Table tennis","key":"act-274"},{"title":"Triathlon","key":"act-273"},{"title":"Weightlifting","key":"act-272"}]},{"title":"Press & Media Comment","key":"act-269"},{"title":"Sponsors","key":"act-270"},{"title":"Competitors","key":"act-266"},{"title":"Cultural Programme","key":"act-295"}]},{"title":"Winter Olympics Sochi 2014","key":"act-177"},{"title":"Oral History in the UK","key":"act-158"},{"title":"Conservative Party Website deletions - Press articles November 2013","key":"act-175"},{"title":"Political Action and Communication","key":"act-159"},{"title":"Tour de France (Yorkshire) 2014","key":"act-264"}] /** * This method computes a tree of collections in JSON format. * * @param targetUrl This is an identifier for current target object * @return tree structure */ @BodyParser.Of(BodyParser.Json.class) public static Result getSuggestedCollections(String targetUrl) { // Logger.debug("getSuggestedCollections() target URL: " + targetUrl); JsonNode jsonData = null; final StringBuffer sb = new StringBuffer(); List<Collection> suggestedCollections = Collection.getFirstLevelCollections(); sb.append(getTreeElements(suggestedCollections, targetUrl, true)); // Logger.debug("collections main level size: " + suggestedCollections.size()); jsonData = Json.toJson(Json.parse(sb.toString())); // Logger.debug("getCollections() json: " + jsonData.toString()); return ok(jsonData); } /** * This method calculates first order collections. * * @param collectionList The list of all collections * @param targetUrl This is an identifier for current target object * @param parent This parameter is used to differentiate between root and children nodes * @return collection object in JSON form */ public static String getTreeElements(List<Collection> collectionList, String targetUrl, boolean parent) { // Logger.debug("getTreeElements() target URL: " + targetUrl); String res = ""; if(collectionList.size() > 0) { final StringBuffer sb = new StringBuffer(); sb.append("["); Iterator<Collection> itr = collectionList.iterator(); boolean firstTime = true; while(itr.hasNext()) { Collection collection = itr.next(); // Logger.debug("add collection: " + collection.name + ", with url: " + collection.url + // ", parent:" + collection.parent + ", parent size: " + collection.parent.length()); if((parent && collection.parent == null) || !parent || collection.parent == null) { if(firstTime) { firstTime = false; } else { sb.append(", "); } // Logger.debug("added"); sb.append("{\"title\": \"" + collection.name + "\"," + checkSelection(collection.url, targetUrl) + " \"key\": \"" + collection.url + "\"" + getChildren(collection.url, targetUrl) + "}"); } } // Logger.debug("collectionList level size: " + collectionList.size()); sb.append("]"); res = sb.toString(); // Logger.debug("getTreeElements() res: " + res); } return res; } /** * This method computes a tree of collections in JSON format for filtering. * * @param targetUrl This is an identifier for current target object * @return tree structure */ @BodyParser.Of(BodyParser.Json.class) public static Result getSuggestedCollectionsFilter(String targetUrl) { // Logger.debug("getSuggestedCollectionsFilter() target URL: " + targetUrl); JsonNode jsonData = null; final StringBuffer sb = new StringBuffer(); List<Collection> suggestedCollections = Collection.getFirstLevelCollections(); sb.append(getTreeElementsFilter(suggestedCollections, targetUrl, true)); // Logger.debug("collections main level size: " + suggestedCollections.size()); jsonData = Json.toJson(Json.parse(sb.toString())); // Logger.debug("getCollections() json: " + jsonData.toString()); return ok(jsonData); } /** * This method filters targets by crawl frequency. * * @return crawl frequency list */ public static List<Target> getCrawlFrequency() { List<Target> res = new ArrayList<Target>(); List<String> subjects = new ArrayList<String>(); List<Target> allTargets = Target.find.all(); Iterator<Target> itr = allTargets.iterator(); while(itr.hasNext()) { Target target = itr.next(); if(target.crawlFrequency != null && target.crawlFrequency.length() > 0 && !subjects.contains(target.crawlFrequency)) { ExpressionList<Target> ll = Target.find.where().contains("crawlFrequency", target.crawlFrequency); if(ll.findRowCount() > 0) { res.add(target); subjects.add(target.crawlFrequency); } } } return res; } public static void raiseFlag(Target target, String flagName) { for(Flag flag : target.flags) { if(flag.name.equals(flagName)) { return; } } target.flags.add(Flag.findByName(flagName)); Ebean.update(target); } private static ObjectNode createCollectionNode(Collection collection) { ObjectNode collectionNode = JsonNodeFactory.instance.objectNode(); Logger.debug("collection: " + collection.name); collectionNode.put("id", collection.id); collectionNode.put("name", collection.name); // collectionNode.put("description", collection.description); collectionNode.put("createdAt", collection.createdAt.toString()); Long milliseconds = collection.updatedAt.getTime(); Long seconds = (Long) (milliseconds / 1000); collectionNode.put("updatedAt", seconds); collectionNode.put("publish", collection.publish); if(collection.parent != null) { collectionNode.put("parent", createCollectionNode(collection.parent)); } return collectionNode; } @Security.Authenticated(SecuredController.class) public static Result getTargetCategories(Long id) { Target target = Target.findById(id); // List<Collection> categories = target.getCollectionCategories(); List<Collection> categories = target.collections; ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode(); for(Collection collection : categories) { ObjectNode collectionNode = createCollectionNode(collection); arrayNode.add(collectionNode); } // JsonNode jsonNode = Json.toJson(categories); // Iterator<JsonNode> iterator = jsonNode.iterator(); // if (categories != null) { // Logger.debug("collections: " + categories.size()); // } // while(iterator.hasNext()) { // JsonNode node = (JsonNode) iterator.next(); // ObjectNode objectNode = (ObjectNode)node; // objectNode.remove("children"); // JsonNode updatedAt = objectNode.get("updatedAt"); // Long milliseconds = updatedAt.asLong(); // Long seconds = (Long)(milliseconds/1000); // Logger.debug("milliseconds: " + milliseconds); // Logger.debug("seconds: " + seconds); // objectNode.put("updatedAt", seconds); // } return ok(arrayNode); } @Security.Authenticated(SecuredController.class) public static Result upload() { User user = User.findByEmail(request().username()); return ok(views.html.targets.upload.render(user)); } @Security.Authenticated(SecuredController.class) public static Result uploadExcel() { MultipartFormData body = request().body().asMultipartFormData(); FilePart picture = body.getFile("excelFile"); if(picture != null) { String fileName = picture.getFilename(); String contentType = picture.getContentType(); File file = picture.getFile(); Logger.info("Uploaded " + fileName); try { excelParser(file); flash("success", "File " + fileName + " uploaded"); } catch(Throwable e) { flash("error", "File " + fileName + " parse errors:"); flash("error_log", e.getMessage()); e.printStackTrace(); } return redirect(routes.TargetController.upload()); } else { Logger.info("Upload failed "); flash("error", "Missing file"); return redirect(routes.TargetController.upload()); } } private static void excelParser(File inputFile) throws Throwable { FileInputStream file = new FileInputStream(inputFile); //Create Workbook instance holding reference to .xls[x] file Workbook workbook = WorkbookFactory.create(file); //Get first/desired sheet from the workbook Sheet sheet = workbook.getSheetAt(0); // Check total row: if(sheet.getPhysicalNumberOfRows() <= 1) { throw new Exception("Sheet should have at least one row."); } Logger.debug("Sheet has " + sheet.getPhysicalNumberOfRows() + " rows."); //Iterate through each rows one by one Iterator<Row> rowIterator = sheet.iterator(); // Header row: Row header = rowIterator.next(); Logger.debug("HEADER: " + header); // TODO Check header row is right. // And the rest: StringBuilder sb = new StringBuilder(); while(rowIterator.hasNext()) { Row row = rowIterator.next(); // Get Target target = new Target(); target.title = row.getCell(0).getStringCellValue(); target.fieldUrls = new ArrayList<FieldUrl>(); // Check URL FieldUrl url = new FieldUrl(row.getCell(1).getStringCellValue()); target.fieldUrls.add(url); FieldUrl existingFieldUrl = FieldUrl.findByUrl(url.url); if(existingFieldUrl != null) { String error = "Row # " + row.getRowNum() + ": CONFLICT - URL " + existingFieldUrl.url + " is already part of target " + existingFieldUrl.target.id + "\n"; Logger.debug(error); sb.append(error); continue; } //Collection c = new Collection(); //c.name = // System.out.println(target); // TODO Merge with controllers.ApplicationController.bulkImport() code to avoid repetition. target.revision = Const.INITIAL_REVISION; target.active = true; target.selectionType = Const.SelectionType.SELECTION.name(); if(target.noLdCriteriaMet == null) { target.noLdCriteriaMet = Boolean.FALSE; } if(target.keySite == null) { target.keySite = Boolean.FALSE; } if(target.ignoreRobotsTxt == null) { target.ignoreRobotsTxt = Boolean.FALSE; } // Save - disabled right now, as we do not want this live as yet. /* target.runChecks(); target.save(); */ // System.out.println(target); } workbook.close(); file.close(); // And report errors if(sb.length() > 0) { throw (new Exception(sb.toString())); } } public static void main(String args[]) { try { excelParser(new File("test/upload/w3act-targets-upload.xlsx")); } catch(Throwable e) { // TODO Auto-generated catch block e.printStackTrace(); } } }