/** * Copyright (C) 2016 Pink Summit, LLC (info@pinksummit.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.di2e.ecdr.commons.endpoint.rest; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.UUID; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.codice.ddf.configuration.SystemBaseUrl; import org.codice.ddf.configuration.SystemInfo; import org.codice.ddf.spatial.geocoder.GeoCoder; import org.codice.ddf.spatial.geocoder.GeoResult; import org.opengis.geometry.BoundingBox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ddf.catalog.CatalogFramework; import ddf.catalog.data.BinaryContent; import ddf.catalog.federation.FederationException; import ddf.catalog.operation.QueryResponse; import ddf.catalog.source.SourceUnavailableException; import ddf.catalog.source.UnsupportedQueryException; import ddf.catalog.transform.CatalogTransformerException; import ddf.registry.api.RegistrableService; import net.di2e.ecdr.api.auditor.SearchAuditor; import net.di2e.ecdr.api.cache.QueryRequestCache; import net.di2e.ecdr.api.query.QueryConfiguration; import net.di2e.ecdr.api.query.QueryCriteria; import net.di2e.ecdr.api.query.QueryLanguage; import net.di2e.ecdr.api.transform.TransformIdMapper; import net.di2e.ecdr.commons.constants.SearchConstants; import net.di2e.ecdr.commons.query.CDRQueryImpl; import net.di2e.ecdr.commons.util.GeospatialUtils; import net.di2e.ecdr.commons.util.SearchUtils; import net.di2e.ecdr.commons.xml.fs.SourceDescription; import net.di2e.ecdr.commons.xml.osd.OpenSearchDescription; import net.di2e.ecdr.commons.xml.osd.Query; import net.di2e.ecdr.commons.xml.osd.SyndicationRight; import net.di2e.ecdr.commons.xml.osd.Url; public abstract class AbstractRestSearchEndpoint implements RegistrableService { private static final Logger LOGGER = LoggerFactory.getLogger( AbstractRestSearchEndpoint.class ); private static Map<String, String> baseQueryParamsMap = null; static { baseQueryParamsMap = new HashMap<String, String>(); baseQueryParamsMap.put( SearchConstants.KEYWORD_PARAMETER, "os:searchTerms" ); baseQueryParamsMap.put( SearchConstants.COUNT_PARAMETER, "os:count" ); baseQueryParamsMap.put( SearchConstants.STARTINDEX_PARAMETER, "os:startIndex" ); baseQueryParamsMap.put( SearchConstants.FORMAT_PARAMETER, "cdrs:responseFormat" ); baseQueryParamsMap.put( SearchConstants.TIMEOUT_PARAMETER, "cdrs:timeout" ); baseQueryParamsMap.put( SearchConstants.STATUS_PARAMETER, "cdrb:includeStatus" ); baseQueryParamsMap.put( SearchConstants.OID_PARAMETER, "cdrsx:originQueryID" ); baseQueryParamsMap.put( SearchConstants.STRICTMODE_PARAMETER, "cdrsx:strictMode" ); baseQueryParamsMap.put( SearchConstants.PATH_PARAMETER, "cdrb:path" ); } private QueryRequestCache queryRequestCache = null; private CatalogFramework catalogFramework = null; private List<SearchAuditor> auditors = null; // private Map<String, QueryLanguage> queryLanguageMap = null; private List<QueryLanguage> queryLanguageList = null; private QueryConfiguration queryConfiguration = null; private TransformIdMapper transformMapper = null; // using an object reference here so that this will be deployable on older DDF systems that do not have the class private List<Object> geoCoderList; /** * Constructor for JAX RS CDR Search Service. Values should ideally be passed into the constructor using a * dependency injection framework like blueprint * * @param framework Catalog Framework which will be used for search * @param queryLangs * @param mapper * @param auditorList * @param queryConfig * @param queryReqCache * @param geoCoderList */ public AbstractRestSearchEndpoint( CatalogFramework framework, List<QueryLanguage> queryLangs, TransformIdMapper mapper, List<SearchAuditor> auditorList, QueryConfiguration queryConfig, QueryRequestCache queryReqCache, List<Object> geoCoderList ) { this.catalogFramework = framework; // this.queryLanguageMap = queryLangs; this.queryLanguageList = queryLangs; this.transformMapper = mapper; this.auditors = auditorList; this.queryConfiguration = queryConfig; this.queryRequestCache = queryReqCache; this.geoCoderList = geoCoderList; } public Response executePing( UriInfo uriInfo, String encodingHeader, String authHeader ) { MultivaluedMap<String, String> queryParams = uriInfo.getQueryParameters(); boolean isValid = isValidQuery( queryParams, SystemInfo.getSiteName() ); return isValid ? Response.ok().build() : Response.status( Response.Status.BAD_REQUEST ).build(); } /** * Search method that gets called when issuing an HTTP GET to the corresponding URL. HTTP GET URL query parameters * contain the query criteria values * * @param uriInfo * Query parameters obtained by e * @param encoding * accept-encoding from the client * @param auth * Authorization header * @return Response to send back to the calling client */ public Response executeSearch( HttpServletRequest servletRequest, UriInfo uriInfo, String encoding, String auth ) { Response response; QueryResponse queryResponse = null; try { String localSourceId = SystemInfo.getSiteName(); MultivaluedMap<String, String> queryParameters = uriInfo.getQueryParameters(); addHeaderParameters( servletRequest, queryParameters ); if ( !isValidQuery( queryParameters, localSourceId ) ) { throw new UnsupportedQueryException( "Invalid query parameters passed in" ); } QueryLanguage queryLanguage = getQueryLanguage( queryParameters ); if ( queryLanguage == null ) { throw new UnsupportedQueryException( "A Query language could not be determined, please check the default query language in the Admin Console ECDR Application Search Endpoint settings" ); } translateGeoNames( queryParameters ); QueryCriteria queryCriteria = queryLanguage.getQueryCriteria( queryParameters, queryConfiguration ); CDRQueryImpl query = new CDRQueryImpl( queryCriteria, localSourceId ); queryResponse = executeQuery( localSourceId, queryParameters, query ); // Move the specific links into Atom Transformer if possible Map<String, Serializable> transformProperties = SearchUtils.getTransformLinkProperties( uriInfo, query, queryResponse, getURLScheme(), SystemBaseUrl.getHost(), Integer.parseInt( SystemBaseUrl.getPort() ) ); transformProperties.put( SearchConstants.FEED_TITLE, "Atom Search Results from '" + localSourceId + "' for Query: " + query.getHumanReadableQuery().trim() ); transformProperties.put( SearchConstants.FORMAT_PARAMETER, query.getResponseFormat() ); transformProperties.put( SearchConstants.STATUS_PARAMETER, isIncludeStatus( queryParameters ) ); transformProperties.put( SearchConstants.LOCAL_SOURCE_ID, catalogFramework.getId() ); transformProperties.put( SearchConstants.GEORSS_RESULT_FORMAT_PARAMETER, getGeoRSSFormat( queryParameters ) ); String format = query.getResponseFormat(); String internalTransformerFormat = transformMapper.getQueryResponseTransformValue( format ); transformProperties.put( SearchConstants.METACARD_TRANSFORMER_NAME, transformMapper.getMetacardTransformValue( format ) ); BinaryContent content = catalogFramework.transform( queryResponse, internalTransformerFormat, transformProperties ); try ( InputStream is = content.getInputStream() ) { response = Response.ok( is, content.getMimeTypeValue() ).build(); } catch ( IOException e ) { LOGGER.error( "Error reading response [" + e.getMessage() + "]", e ); response = Response.status( Response.Status.INTERNAL_SERVER_ERROR ).build(); } } catch ( UnsupportedQueryException e ) { LOGGER.error( e.getMessage(), e ); response = Response.status( Response.Status.BAD_REQUEST ).build(); } catch ( SourceUnavailableException e ) { LOGGER.error( e.getMessage(), e ); response = Response.status( Response.Status.INTERNAL_SERVER_ERROR ).build(); } catch ( FederationException e ) { LOGGER.error( e.getMessage(), e ); response = Response.status( Response.Status.BAD_REQUEST ).build(); // These exceptions happen when the transform is not available via // the framework or an exception occurs in translation } catch ( CatalogTransformerException | IllegalArgumentException e ) { LOGGER.error( e.getMessage(), e ); response = Response.status( Response.Status.BAD_REQUEST ).build(); } catch ( RuntimeException e ) { LOGGER.error( "Unexpected exception received [" + e.getMessage() + "]", e ); response = Response.status( Response.Status.INTERNAL_SERVER_ERROR ).build(); } for ( SearchAuditor auditor : auditors ) { auditor.auditRESTQuery( servletRequest, queryResponse, response ); } return response; } @GET @Path( "/osd.xml" ) @Produces( "application/opensearchdescription+xml" ) public Response getOSD() { OpenSearchDescription osd = new OpenSearchDescription(); osd.setShortName( SystemInfo.getSiteName() ); osd.setDescription( getServiceDescription() ); osd.setTags( "ecdr opensearch cdr ddf" ); if ( StringUtils.isNotBlank( SystemInfo.getOrganization() ) ) { osd.setDeveloper( SystemInfo.getOrganization() ); } if ( StringUtils.isNotBlank( SystemInfo.getSiteContatct() ) ) { osd.setContact( SystemInfo.getSiteContatct() ); } Query query = new Query(); query.setRole( "example" ); query.setSearchTerms( "test" ); osd.getQuery().add( query ); osd.setSyndicationRight( SyndicationRight.OPEN ); osd.getLanguage().add( MediaType.MEDIA_TYPE_WILDCARD ); osd.getInputEncoding().add( StandardCharsets.UTF_8.name() ); osd.getOutputEncoding().add( StandardCharsets.UTF_8.name() ); // url example for ( QueryLanguage lang : queryLanguageList ) { Url url = new Url(); url.setType( MediaType.APPLICATION_ATOM_XML ); url.setTemplate( generateTemplateUrl( lang ) ); osd.getUrl().add( url ); } addSourceDescriptions( osd ); StringWriter writer = new StringWriter(); InputStream is = null; try { JAXBContext context = JAXBContext.newInstance( OpenSearchDescription.class, SourceDescription.class ); Marshaller marshaller = context.createMarshaller(); marshaller.setProperty( Marshaller.JAXB_FORMATTED_OUTPUT, true ); marshaller.setProperty( Marshaller.JAXB_FRAGMENT, true ); marshaller.marshal( osd, writer ); is = getClass().getResourceAsStream( "/templates/osd_info.template" ); if ( is != null ) { String osdTemplate = IOUtils.toString( is ); osdTemplate = replaceTemplateValues( osdTemplate ); String responseStr = osdTemplate + writer.toString(); return Response.ok( responseStr, MediaType.APPLICATION_XML_TYPE ).build(); } else { return Response.serverError().entity( "COULD NOT LOAD OSD TEMPLATE." ).build(); } } catch ( JAXBException | IOException e ) { LOGGER.warn( "Could not create OSD for client due to exception.", e ); return Response.serverError().build(); } finally { IOUtils.closeQuietly( is ); } } protected QueryLanguage getQueryLanguage( MultivaluedMap<String, String> queryParams ) { String lang = StringUtils.defaultIfBlank( queryParams.getFirst( SearchConstants.QUERYLANGUAGE_PARAMETER ), queryConfiguration.getDefaultQueryLanguage() ); LOGGER.debug( "Using query language that is associated with the name [{}]", lang ); for ( QueryLanguage queryLang : queryLanguageList ) { if ( StringUtils.equalsIgnoreCase( queryLang.getName(), lang ) ) { return queryLang; } } return null; } private void translateGeoNames( MultivaluedMap<String, String> queryParameters ) { String geoName = queryParameters.getFirst( SearchConstants.GEO_NAME_PARAMETER ); if ( StringUtils.isNotBlank( geoName )) { for (Object curObject : geoCoderList) { GeoCoder geoCoder = (GeoCoder) curObject; GeoResult result = geoCoder.getLocation( geoName ); if (result != null) { if (result.getBbox() != null) { BoundingBox boundingBox = GeospatialUtils.pointsToBBox(result.getBbox()); if (boundingBox != null) { String wktStr = GeospatialUtils.bboxToWKT(boundingBox); queryParameters.add( SearchConstants.GEOMETRY_PARAMETER, wktStr ); } else { LOGGER.debug("Was not able to convert geoname result to boundingbox, checking next geocoder."); continue; } } else if (result.getPoint() != null) { String wktStr = GeospatialUtils.pointToWKT(result.getPoint()); queryParameters.add( SearchConstants.GEOMETRY_PARAMETER, wktStr ); } else { // issue within the geocoder, it had a result but nothing converted in it continue; } return; } } } } protected void addSourceDescriptions( OpenSearchDescription osd ) { // federated sites for ( String curSource : catalogFramework.getSourceIds() ) { SourceDescription description = new SourceDescription(); description.setSourceId( curSource ); description.setShortName( curSource ); osd.getAny().add( description ); } } protected String replaceTemplateValues( String osdTemplate ) { osdTemplate = StringUtils.replace( osdTemplate, "${defaultCount}", String.valueOf( queryConfiguration.getDefaultCount() ), 1 ); osdTemplate = StringUtils.replace( osdTemplate, "${defaultQueryLanguage}", queryConfiguration.getDefaultQueryLanguage(), 1 ); osdTemplate = StringUtils.replace( osdTemplate, "${queryLanguages}", getQueryLanguagesString(), 1 ); osdTemplate = StringUtils.replace( osdTemplate, "${defaultResponseFormat}", queryConfiguration.getDefaultResponseFormat(), 1 ); osdTemplate = StringUtils.replace( osdTemplate, "${defaultTimeout}", String.valueOf( queryConfiguration.getDefaultTimeoutMillis() ), 1 ); osdTemplate = StringUtils.replace( osdTemplate, "${additionalBasicParameters}", "", 1 ); osdTemplate = StringUtils.replace( osdTemplate, "${queryLanguageDocumentation}", getQueryLanguageDescriptions(), 1 ); return osdTemplate; } protected String getQueryLanguageDescriptions() { StringBuilder sb = new StringBuilder(); Iterator<QueryLanguage> langIter = queryLanguageList.iterator(); while ( langIter.hasNext() ) { sb.append( langIter.next().getLanguageDescription( queryConfiguration ) ); if ( langIter.hasNext() ) { sb.append( System.lineSeparator() ); sb.append( System.lineSeparator() ); sb.append( System.lineSeparator() ); } } return sb.toString(); } protected void addHeaderParameters( HttpServletRequest servletRequest, MultivaluedMap<String, String> queryParameters ) { List<String> headerProperties = queryConfiguration.getHeaderPropertyList(); if ( CollectionUtils.isNotEmpty( headerProperties ) ) { for ( String headerProp : headerProperties ) { String value = servletRequest.getHeader( headerProp ); if ( StringUtils.isNotBlank( value ) ) { LOGGER.trace( "Matching HTTP Header key/value pair found, adding [{}]=[{}] to queryParameters", headerProp, value ); queryParameters.putSingle( headerProp, value ); } } } } private String getQueryLanguagesString() { StringBuilder builder = new StringBuilder(); for ( QueryLanguage lang : queryLanguageList ) { builder.append( "'" + lang.getName() + "' " ); } return builder.toString().trim(); } public String getParameterTemplate( String languageName ) { StringBuilder sb = new StringBuilder( "?" ); for ( Entry<String, String> entry : baseQueryParamsMap.entrySet() ) { sb.append( entry.getKey() + "={" + entry.getValue() + "?}&" ); } // Query Language isn't listed in the default set of values sb.append( SearchConstants.QUERYLANGUAGE_PARAMETER + "=" + languageName ); return sb.toString(); } public abstract QueryResponse executeQuery( String localSourceId, MultivaluedMap<String, String> queryParameters, CDRQueryImpl query ) throws SourceUnavailableException, UnsupportedQueryException, FederationException; @Override public Map<String, String> getProperties() { return Collections.emptyMap(); } protected CatalogFramework getCatalogFramework() { return catalogFramework; } protected boolean isIncludeStatus( MultivaluedMap<String, String> queryParameters ) { // Include status is true unless explicitly set to false return BooleanUtils.toBooleanDefaultIfNull( SearchUtils.getBoolean( queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ) ), true ); } protected String getGeoRSSFormat( MultivaluedMap<String, String> queryParameters ) { return StringUtils.defaultIfBlank( queryParameters.getFirst( SearchConstants.GEORSS_RESULT_FORMAT_PARAMETER ), null ); } public Map<String, Serializable> getQueryProperties( MultivaluedMap<String, String> queryParameters, String sourceId ) { Map<String, Serializable> queryProperties = new HashMap<String, Serializable>(); queryProperties.put( SearchConstants.FORMAT_PARAMETER, StringUtils.defaultIfBlank( queryParameters.getFirst( SearchConstants.FORMAT_PARAMETER ), queryConfiguration.getDefaultResponseFormat() ) ); queryProperties.put( SearchConstants.STATUS_PARAMETER, SearchUtils.getBoolean( queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ), true ) ); queryProperties.put( SearchConstants.DEDUP_PARAMETER, SearchUtils.getBoolean( queryParameters.getFirst( SearchConstants.DEDUP_PARAMETER ), queryConfiguration.isDefaultDeduplication() ) ); for ( String key : queryParameters.keySet() ) { String value = queryParameters.getFirst( key ); if ( StringUtils.isNotBlank( value ) && (queryConfiguration.getParameterPropertyList().contains( key ) || queryConfiguration.getHeaderPropertyList().contains( key )) ) { LOGGER.trace( "Adding key/value pair [{}]=[{}] to queryProperties that get sent in with query request", key, value ); queryProperties.put( key, value ); } } LOGGER.trace( "Setting the query properties to {} based on values in query parameters {}", queryProperties, queryParameters ); return queryProperties; } protected String generateTemplateUrl( QueryLanguage lang ) { StringBuilder urlBuilder = new StringBuilder(); urlBuilder.append( SystemBaseUrl.getProtocol() ); urlBuilder.append( SystemBaseUrl.getHost() ); urlBuilder.append( ":" ); urlBuilder.append( SystemBaseUrl.getPort() ); urlBuilder.append( getServiceRelativeUrl() ); urlBuilder.append( getParameterTemplate( lang.getName() ) ); urlBuilder.append( lang.getUrlTemplateParameters() ); LOGGER.debug( "Generating the following template URL for OSDD: {}", urlBuilder ); return urlBuilder.toString(); } protected boolean isValidQuery( MultivaluedMap<String, String> queryParameters, String sourceId ) { boolean isValidQuery; String queryLang = queryParameters.getFirst( SearchConstants.QUERYLANGUAGE_PARAMETER ); // if ( StringUtils.isNotBlank( queryLang ) && !queryLanguageMap.containsKey( queryLang ) ) { if ( getQueryLanguage( queryParameters ) == null ) { isValidQuery = false; LOGGER.debug( "The query is not valid because the {} parameter with value {} is not in the allowed values {}", SearchConstants.QUERYLANGUAGE_PARAMETER, queryLang, queryLanguageList ); } else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.CASESENSITIVE_PARAMETER ) ) ) { isValidQuery = false; LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.CASESENSITIVE_PARAMETER, queryParameters.getFirst( SearchConstants.CASESENSITIVE_PARAMETER ) ); } else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.STRICTMODE_PARAMETER ) ) ) { isValidQuery = false; LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.STRICTMODE_PARAMETER, queryParameters.getFirst( SearchConstants.STRICTMODE_PARAMETER ) ); } else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ) ) ) { isValidQuery = false; LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.STATUS_PARAMETER, queryParameters.getFirst( SearchConstants.STATUS_PARAMETER ) ); } else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.FUZZY_PARAMETER ) ) ) { isValidQuery = false; LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.FUZZY_PARAMETER, queryParameters.getFirst( SearchConstants.FUZZY_PARAMETER ) ); } else if ( !SearchUtils.isBooleanNullOrBlank( queryParameters.getFirst( SearchConstants.DEDUP_PARAMETER ) ) ) { isValidQuery = false; LOGGER.debug( "The query is not valid because the {} parameter with value {} is not valid", SearchConstants.DEDUP_PARAMETER, queryParameters.getFirst( SearchConstants.DEDUP_PARAMETER ) ); } else { isValidQuery = isUniqueQuery( queryParameters, sourceId ); LOGGER.debug( "Checking if the query is valid: {}", isValidQuery ); } return isValidQuery; } protected boolean isUniqueQuery( MultivaluedMap<String, String> queryParameters, String sourceId ) { boolean isUniqueQuery = true; String oid = queryParameters.getFirst( SearchConstants.OID_PARAMETER ); if ( StringUtils.isNotBlank( oid ) ) { isUniqueQuery = queryRequestCache.isQueryIdUnique( oid ); } else { String uuid = UUID.randomUUID().toString(); queryParameters.putSingle( SearchConstants.OID_PARAMETER, uuid ); queryRequestCache.add( uuid ); } String path = queryParameters.getFirst( SearchConstants.PATH_PARAMETER ); if ( StringUtils.isNotBlank( path ) ) { String[] pathValues = path.split( "," ); if ( ArrayUtils.contains( pathValues, sourceId ) ) { isUniqueQuery = false; LOGGER.debug( "The '{}' with value '{}' contains the local source id {}", SearchConstants.PATH_PARAMETER, path, sourceId ); } } else { queryParameters.putSingle( SearchConstants.PATH_PARAMETER, catalogFramework.getId() ); } return isUniqueQuery; } protected String getURLScheme() { return StringUtils.substringBefore( SystemBaseUrl.getProtocol(), ":" ); } }