/**
* Copyright (C) 2016 Pink Summit, LLC (info@pinksummit.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.di2e.ecdr.commons.endpoint.rest;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.UUID;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.BooleanUtils;
import org.apache.commons.lang.StringUtils;
import org.codice.ddf.configuration.SystemBaseUrl;
import org.codice.ddf.configuration.SystemInfo;
import org.codice.ddf.spatial.geocoder.GeoCoder;
import org.codice.ddf.spatial.geocoder.GeoResult;
import org.opengis.geometry.BoundingBox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ddf.catalog.CatalogFramework;
import ddf.catalog.data.BinaryContent;
import ddf.catalog.federation.FederationException;
import ddf.catalog.operation.QueryResponse;
import ddf.catalog.source.SourceUnavailableException;
import ddf.catalog.source.UnsupportedQueryException;
import ddf.catalog.transform.CatalogTransformerException;
import ddf.registry.api.RegistrableService;
import net.di2e.ecdr.api.auditor.SearchAuditor;
import net.di2e.ecdr.api.cache.QueryRequestCache;
import net.di2e.ecdr.api.query.QueryConfiguration;
import net.di2e.ecdr.api.query.QueryCriteria;
import net.di2e.ecdr.api.query.QueryLanguage;
import net.di2e.ecdr.api.transform.TransformIdMapper;
import net.di2e.ecdr.commons.constants.SearchConstants;
import net.di2e.ecdr.commons.query.CDRQueryImpl;
import net.di2e.ecdr.commons.util.GeospatialUtils;
import net.di2e.ecdr.commons.util.SearchUtils;
import net.di2e.ecdr.commons.xml.fs.SourceDescription;
import net.di2e.ecdr.commons.xml.osd.OpenSearchDescription;
import net.di2e.ecdr.commons.xml.osd.Query;
import net.di2e.ecdr.commons.xml.osd.SyndicationRight;
import net.di2e.ecdr.commons.xml.osd.Url;
public abstract class AbstractRestSearchEndpoint implements RegistrableService {
private static final Logger LOGGER = LoggerFactory.getLogger( AbstractRestSearchEndpoint.class );
private static Map<String, String> baseQueryParamsMap = null;
static {
baseQueryParamsMap = new HashMap<String, String>();
baseQueryParamsMap.put( SearchConstants.KEYWORD_PARAMETER, "os:searchTerms" );
baseQueryParamsMap.put( SearchConstants.COUNT_PARAMETER, "os:count" );
baseQueryParamsMap.put( SearchConstants.STARTINDEX_PARAMETER, "os:startIndex" );
baseQueryParamsMap.put( SearchConstants.FORMAT_PARAMETER, "cdrs:responseFormat" );
baseQueryParamsMap.put( SearchConstants.TIMEOUT_PARAMETER, "cdrs:timeout" );
baseQueryParamsMap.put( SearchConstants.STATUS_PARAMETER, "cdrb:includeStatus" );
baseQueryParamsMap.put( SearchConstants.OID_PARAMETER, "cdrsx:originQueryID" );
baseQueryParamsMap.put( SearchConstants.STRICTMODE_PARAMETER, "cdrsx:strictMode" );
baseQueryParamsMap.put( SearchConstants.PATH_PARAMETER, "cdrb:path" );
}
private QueryRequestCache queryRequestCache = null;
private CatalogFramework catalogFramework = null;
private List<SearchAuditor> auditors = null;
// private Map<String, QueryLanguage> queryLanguageMap = null;
private List<QueryLanguage> queryLanguageList = null;
private QueryConfiguration queryConfiguration = null;
private TransformIdMapper transformMapper = null;
// using an object reference here so that this will be deployable on older DDF systems that do not have the class
private List<Object> geoCoderList;
/**
* Constructor for JAX RS CDR Search Service. Values should ideally be passed into the constructor using a
* dependency injection framework like blueprint
*
* @param framework Catalog Framework which will be used for search
* @param queryLangs
* @param mapper
* @param auditorList
* @param queryConfig
* @param queryReqCache
* @param geoCoderList
*/
public AbstractRestSearchEndpoint( CatalogFramework framework, List<QueryLanguage> queryLangs, TransformIdMapper mapper, List<SearchAuditor> auditorList,
QueryConfiguration queryConfig, QueryRequestCache queryReqCache, List<Object> geoCoderList ) {
this.catalogFramework = framework;
// this.queryLanguageMap = queryLangs;
this.queryLanguageList = queryLangs;
this.transformMapper = mapper;
this.auditors = auditorList;
this.queryConfiguration = queryConfig;
this.queryRequestCache = queryReqCache;
this.geoCoderList = geoCoderList;
}
public Response executePing( UriInfo uriInfo, String encodingHeader, String authHeader ) {
MultivaluedMap<String, String> queryParams = uriInfo.getQueryParameters();
boolean isValid = isValidQuery( queryParams, SystemInfo.getSiteName() );
return isValid ? Response.ok().build() : Response.status( Response.Status.BAD_REQUEST ).build();
}
/**
* Search method that gets called when issuing an HTTP GET to the corresponding URL. HTTP GET URL query parameters
* contain the query criteria values
*
* @param uriInfo
* Query parameters obtained by e
* @param encoding
* accept-encoding from the client
* @param auth
* Authorization header
* @return Response to send back to the calling client
*/
public Response executeSearch( HttpServletRequest servletRequest, UriInfo uriInfo, String encoding, String auth ) {
Response response;
QueryResponse queryResponse = null;
try {
String localSourceId = SystemInfo.getSiteName();
MultivaluedMap<String, String> queryParameters = uriInfo.getQueryParameters();
addHeaderParameters( servletRequest, queryParameters );
if ( !isValidQuery( queryParameters, localSourceId ) ) {
throw new UnsupportedQueryException( "Invalid query parameters passed in" );
}
QueryLanguage queryLanguage = getQueryLanguage( queryParameters );
if ( queryLanguage == null ) {
throw new UnsupportedQueryException(
"A Query language could not be determined, please check the default query language in the Admin Console ECDR Application Search Endpoint settings" );
}
translateGeoNames( queryParameters );
QueryCriteria queryCriteria = queryLanguage.getQueryCriteria( queryParameters, queryConfiguration );
CDRQueryImpl query = new CDRQueryImpl( queryCriteria, localSourceId );
queryResponse = executeQuery( localSourceId, queryParameters, query );
// Move the specific links into Atom Transformer if possible
Map<String, Serializable> transformProperties = SearchUtils.getTransformLinkProperties( uriInfo, query, queryResponse, getURLScheme(),
SystemBaseUrl.getHost(), Integer.parseInt( SystemBaseUrl.getPort() ) );
transformProperties.put( SearchConstants.FEED_TITLE, "Atom Search Results from '" + localSourceId + "' for Query: " + query.getHumanReadableQuery().trim() );
transformProperties.put( SearchConstants.FORMAT_PARAMETER, query.getResponseFormat() );
transformProperties.put( SearchConstants.STATUS_PARAMETER, isIncludeStatus( queryParameters ) );
transformProperties.put( SearchConstants.LOCAL_SOURCE_ID, catalogFramework.getId() );
transformProperties.put( SearchConstants.GEORSS_RESULT_FORMAT_PARAMETER, getGeoRSSFormat( queryParameters ) );
String format = query.getResponseFormat();
String internalTransformerFormat = transformMapper.getQueryResponseTransformValue( format );
transformProperties.put( SearchConstants.METACARD_TRANSFORMER_NAME, transformMapper.getMetacardTransformValue( format ) );
BinaryContent content = catalogFramework.transform( queryResponse, internalTransformerFormat, transformProperties );
try ( InputStream is = content.getInputStream() ) {
response = Response.ok( is, content.getMimeTypeValue() ).build();
} catch ( IOException e ) {
LOGGER.error( "Error reading response [" + e.getMessage() + "]", e );
response = Response.status( Response.Status.INTERNAL_SERVER_ERROR ).build();
}
} catch ( UnsupportedQueryException e ) {
LOGGER.error( e.getMessage(), e );
response = Response.status( Response.Status.BAD_REQUEST ).build();
} catch ( SourceUnavailableException e ) {
LOGGER.error( e.getMessage(), e );
response = Response.status( Response.Status.INTERNAL_SERVER_ERROR ).build();
} catch ( FederationException e ) {
LOGGER.error( e.getMessage(), e );
response = Response.status( Response.Status.BAD_REQUEST ).build();
// These exceptions happen when the transform is not available via
// the framework or an exception occurs in translation
} catch ( CatalogTransformerException | IllegalArgumentException e ) {
LOGGER.error( e.getMessage(), e );
response = Response.status( Response.Status.BAD_REQUEST ).build();
} catch ( RuntimeException e ) {
LOGGER.error( "Unexpected exception received [" + e.getMessage() + "]", e );
response = Response.status( Response.Status.INTERNAL_SERVER_ERROR ).build();
}
for ( SearchAuditor auditor : auditors ) {
auditor.auditRESTQuery( servletRequest, queryResponse, response );
}
return response;
}
@GET
@Path( "/osd.xml" )
@Produces( "application/opensearchdescription+xml" )
public Response getOSD() {
OpenSearchDescription osd = new OpenSearchDescription();
osd.setShortName( SystemInfo.getSiteName() );
osd.setDescription( getServiceDescription() );
osd.setTags( "ecdr opensearch cdr ddf" );
if ( StringUtils.isNotBlank( SystemInfo.getOrganization() ) ) {
osd.setDeveloper( SystemInfo.getOrganization() );
}
if ( StringUtils.isNotBlank( SystemInfo.getSiteContatct() ) ) {
osd.setContact( SystemInfo.getSiteContatct() );
}
Query query = new Query();
query.setRole( "example" );
query.setSearchTerms( "test" );
osd.getQuery().add( query );
osd.setSyndicationRight( SyndicationRight.OPEN );
osd.getLanguage().add( MediaType.MEDIA_TYPE_WILDCARD );
osd.getInputEncoding().add( StandardCharsets.UTF_8.name() );
osd.getOutputEncoding().add( StandardCharsets.UTF_8.name() );
// url example
for ( QueryLanguage lang : queryLanguageList ) {
Url url = new Url();
url.setType( MediaType.APPLICATION_ATOM_XML );
url.setTemplate( generateTemplateUrl( lang ) );
osd.getUrl().add( url );
}
addSourceDescriptions( osd );
StringWriter writer = new StringWriter();
InputStream is = null;
try {
JAXBContext context = JAXBContext.newInstance( OpenSearchDescription.class, SourceDescription.class );
Marshaller marshaller = context.createMarshaller();
marshaller.setProperty( Marshaller.JAXB_FORMATTED_OUTPUT, true );
marshaller.setProperty( Marshaller.JAXB_FRAGMENT, true );
marshaller.marshal( osd, writer );
is = getClass().getResourceAsStream( "/templates/osd_info.template" );
if ( is != null ) {
String osdTemplate = IOUtils.toString( is );
osdTemplate = replaceTemplateValues( osdTemplate );
String responseStr = osdTemplate + writer.toString();
return Response.ok( responseStr, MediaType.APPLICATION_XML_TYPE ).build();
} else {
return Response.serverError().entity( "COULD NOT LOAD OSD TEMPLATE." ).build();
}
} catch ( JAXBException | IOException e ) {
LOGGER.warn( "Could not create OSD for client due to exception.", e );
return Response.serverError().build();
} finally {
IOUtils.closeQuietly( is );
}
}
protected QueryLanguage getQueryLanguage( MultivaluedMap<String, String> queryParams ) {
String lang = StringUtils.defaultIfBlank( queryParams.getFirst( SearchConstants.QUERYLANGUAGE_PARAMETER ), queryConfiguration.getDefaultQueryLanguage() );
LOGGER.debug( "Using query language that is associated with the name [{}]", lang );
for ( QueryLanguage queryLang : queryLanguageList ) {
if ( StringUtils.equalsIgnoreCase( queryLang.getName(), lang ) ) {
return queryLang;
}
}
return null;
}
private void translateGeoNames( MultivaluedMap<String, String> queryParameters ) {
String geoName = queryParameters.getFirst( SearchConstants.GEO_NAME_PARAMETER );
if ( StringUtils.isNotBlank( geoName )) {
for (Object curObject : geoCoderList) {
GeoCoder geoCoder = (GeoCoder) curObject;
GeoResult result = geoCoder.getLocation( geoName );
if (result != null) {
if (result.getBbox() != null) {
BoundingBox boundingBox = GeospatialUtils.pointsToBBox(result.getBbox());
if (boundingBox != null) {
String wktStr = GeospatialUtils.bboxToWKT(boundingBox);
queryParameters.add( SearchConstants.GEOMETRY_PARAMETER, wktStr );
} else {
LOGGER.debug("Was not able to convert geoname result to boundingbox, checking next geocoder.");
continue;
}
} else if (result.getPoint() != null) {
String wktStr = GeospatialUtils.pointToWKT(result.getPoint());
queryParameters.add( SearchConstants.GEOMETRY_PARAMETER, wktStr );
} else {
// issue within the geocoder, it had a result but nothing converted in it
continue;
}
return;
}
}
}
}
protected void addSourceDescriptions( OpenSearchDescription osd ) {
// federated sites
for ( String curSource : catalogFramework.getSourceIds() ) {
SourceDescription description = new SourceDescription();
description.setSourceId( curSource );
description.setShortName( curSource );
osd.getAny().add( description );
}
}
protected String replaceTemplateValues( String osdTemplate ) {
osdTemplate = StringUtils.replace( osdTemplate, "${defaultCount}", String.valueOf( queryConfiguration.getDefaultCount() ), 1 );
osdTemplate = StringUtils.replace( osdTemplate, "${defaultQueryLanguage}", queryConfiguration.getDefaultQueryLanguage(), 1 );
osdTemplate = StringUtils.replace( osdTemplate, "${queryLanguages}", getQueryLanguagesString(), 1 );
osdTemplate = StringUtils.replace( osdTemplate, "${defaultResponseFormat}", queryConfiguration.getDefaultResponseFormat(), 1 );
osdTemplate = StringUtils.replace( osdTemplate, "${defaultTimeout}", String.valueOf( queryConfiguration.getDefaultTimeoutMillis() ), 1 );
osdTemplate = StringUtils.replace( osdTemplate, "${additionalBasicParameters}", "", 1 );
osdTemplate = StringUtils.replace( osdTemplate, "${queryLanguageDocumentation}", getQueryLanguageDescriptions(), 1 );
return osdTemplate;
}
protected String getQueryLanguageDescriptions() {
StringBuilder sb = new StringBuilder();
Iterator<QueryLanguage> langIter = queryLanguageList.iterator();
while ( langIter.hasNext() ) {
sb.append( langIter.next().getLanguageDescription( queryConfiguration ) );
if ( langIter.hasNext() ) {
sb.append( System.lineSeparator() );
sb.append( System.lineSeparator() );
sb.append( System.lineSeparator() );
}
}
return sb.toString();
}
protected void addHeaderParameters( HttpServletRequest servletRequest, MultivaluedMap<String, String> queryParameters ) {
List<String> headerProperties = queryConfiguration.getHeaderPropertyList();
if ( CollectionUtils.isNotEmpty( headerProperties ) ) {
for ( String headerProp : headerProperties ) {
String value = servletRequest.getHeader( headerProp );
if ( StringUtils.isNotBlank( value ) ) {
LOGGER.trace( "Matching HTTP Header key/value pair found, adding [{}]=[{}] to queryParameters", headerProp, value );
queryParameters.putSingle( headerProp, value );
}
}
}
}
private String getQueryLanguagesString() {
StringBuilder builder = new StringBuilder();
for ( QueryLanguage lang : queryLanguageList ) {
builder.append( "'" + lang.getName() + "' " );
}
return builder.toString().trim();
}
public String getParameterTemplate( String languageName ) {
StringBuilder sb = new StringBuilder( "?" );
for ( Entry<String, String> entry : baseQueryParamsMap.entrySet() ) {
sb.append( entry.getKey() + "={" + entry.getValue() + "?}&" );
}
// Query Language isn't listed in the default set of values
sb.append( SearchConstants.QUERYLANGUAGE_PARAMETER + "=" + languageName );
return sb.toString();
}
public abstract QueryResponse executeQuery( String localSourceId, MultivaluedMap<String, String> queryParameters, CDRQueryImpl query ) throws SourceUnavailableException,
UnsupportedQueryException, FederationException;
@Override
public Map<String, String> getProperties() {
return Collections.emptyMap();
}
protected CatalogFramework getCatalogFramework() {
return catalogFramework;
}
protected boolean isIncludeStatus( MultivaluedMap<String, String> queryParameters ) {
// Include status is true unless explicitly set to false
return BooleanUtils.toBooleanDefaultIfNull( SearchUtils.getBoolean( queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ) ), true );
}
protected String getGeoRSSFormat( MultivaluedMap<String, String> queryParameters ) {
return StringUtils.defaultIfBlank( queryParameters.getFirst( SearchConstants.GEORSS_RESULT_FORMAT_PARAMETER ), null );
}
public Map<String, Serializable> getQueryProperties( MultivaluedMap<String, String> queryParameters, String sourceId ) {
Map<String, Serializable> queryProperties = new HashMap<String, Serializable>();
queryProperties.put( SearchConstants.FORMAT_PARAMETER,
StringUtils.defaultIfBlank( queryParameters.getFirst( SearchConstants.FORMAT_PARAMETER ), queryConfiguration.getDefaultResponseFormat() ) );
queryProperties.put( SearchConstants.STATUS_PARAMETER, SearchUtils.getBoolean( queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ), true ) );
queryProperties.put( SearchConstants.DEDUP_PARAMETER, SearchUtils.getBoolean( queryParameters.getFirst( SearchConstants.DEDUP_PARAMETER ), queryConfiguration.isDefaultDeduplication() ) );
for ( String key : queryParameters.keySet() ) {
String value = queryParameters.getFirst( key );
if ( StringUtils.isNotBlank( value ) && (queryConfiguration.getParameterPropertyList().contains( key ) || queryConfiguration.getHeaderPropertyList().contains( key )) ) {
LOGGER.trace( "Adding key/value pair [{}]=[{}] to queryProperties that get sent in with query request", key, value );
queryProperties.put( key, value );
}
}
LOGGER.trace( "Setting the query properties to {} based on values in query parameters {}", queryProperties, queryParameters );
return queryProperties;
}
protected String generateTemplateUrl( QueryLanguage lang ) {
StringBuilder urlBuilder = new StringBuilder();
urlBuilder.append( SystemBaseUrl.getProtocol() );
urlBuilder.append( SystemBaseUrl.getHost() );
urlBuilder.append( ":" );
urlBuilder.append( SystemBaseUrl.getPort() );
urlBuilder.append( getServiceRelativeUrl() );
urlBuilder.append( getParameterTemplate( lang.getName() ) );
urlBuilder.append( lang.getUrlTemplateParameters() );
LOGGER.debug( "Generating the following template URL for OSDD: {}", urlBuilder );
return urlBuilder.toString();
}
protected boolean isValidQuery( MultivaluedMap<String, String> queryParameters, String sourceId ) {
boolean isValidQuery;
String queryLang = queryParameters.getFirst( SearchConstants.QUERYLANGUAGE_PARAMETER );
// if ( StringUtils.isNotBlank( queryLang ) && !queryLanguageMap.containsKey( queryLang ) ) {
if ( getQueryLanguage( queryParameters ) == null ) {
isValidQuery = false;
LOGGER.debug( "The query is not valid because the {} parameter with value {} is not in the allowed values {}", SearchConstants.QUERYLANGUAGE_PARAMETER, queryLang, queryLanguageList );
} else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.CASESENSITIVE_PARAMETER ) ) ) {
isValidQuery = false;
LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.CASESENSITIVE_PARAMETER,
queryParameters.getFirst( SearchConstants.CASESENSITIVE_PARAMETER ) );
} else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.STRICTMODE_PARAMETER ) ) ) {
isValidQuery = false;
LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.STRICTMODE_PARAMETER,
queryParameters.getFirst( SearchConstants.STRICTMODE_PARAMETER ) );
} else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ) ) ) {
isValidQuery = false;
LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.STATUS_PARAMETER,
queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ) );
} else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.FUZZY_PARAMETER ) ) ) {
isValidQuery = false;
LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.FUZZY_PARAMETER, queryParameters.getFirst( SearchConstants.FUZZY_PARAMETER ) );
} else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.DEDUP_PARAMETER ) ) ) {
isValidQuery = false;
LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.DEDUP_PARAMETER, queryParameters.getFirst( SearchConstants.DEDUP_PARAMETER ) );
} else {
isValidQuery = isUniqueQuery( queryParameters, sourceId );
LOGGER.debug( "Checking if the query is valid: {}", isValidQuery );
}
return isValidQuery;
}
protected boolean isUniqueQuery( MultivaluedMap<String, String> queryParameters, String sourceId ) {
boolean isUniqueQuery = true;
String oid = queryParameters.getFirst( SearchConstants.OID_PARAMETER );
if ( StringUtils.isNotBlank( oid ) ) {
isUniqueQuery = queryRequestCache.isQueryIdUnique( oid );
} else {
String uuid = UUID.randomUUID().toString();
queryParameters.putSingle( SearchConstants.OID_PARAMETER, uuid );
queryRequestCache.add( uuid );
}
String path = queryParameters.getFirst( SearchConstants.PATH_PARAMETER );
if ( StringUtils.isNotBlank( path ) ) {
String[] pathValues = path.split( "," );
if ( ArrayUtils.contains( pathValues, sourceId ) ) {
isUniqueQuery = false;
LOGGER.debug( "The '{}' with value '{}' contains the local source id {}", SearchConstants.PATH_PARAMETER, path, sourceId );
}
} else {
queryParameters.putSingle( SearchConstants.PATH_PARAMETER, catalogFramework.getId() );
}
return isUniqueQuery;
}
protected String getURLScheme() {
return StringUtils.substringBefore( SystemBaseUrl.getProtocol(), ":" );
}
}