OpenSearchServlet.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.nutch.webapp.servlet;

import java.io.IOException;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.nutch.searcher.Hit;
import org.apache.nutch.searcher.HitDetails;
import org.apache.nutch.searcher.Hits;
import org.apache.nutch.searcher.NutchBean;
import org.apache.nutch.searcher.Query;
import org.apache.nutch.searcher.Summary;
import org.w3c.dom.Attr;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;

/**
 * Present search results using A9's OpenSearch extensions to RSS, plus a few
 * Nutch-specific extensions.
 */
public class OpenSearchServlet extends NutchHttpServlet {
  private static final int DEFAULT_NUMBER_OF_HITS = 10;

  private static final long serialVersionUID = 1L;

  private static final Map NS_MAP = new HashMap();

  static {
    NS_MAP.put("opensearch", "http://a9.com/-/spec/opensearchrss/1.0/");
    NS_MAP.put("nutch", "http://www.nutch.org/opensearchrss/1.0/");
  }

  private static final Set SKIP_DETAILS = new HashSet();
  static {
    SKIP_DETAILS.add("url"); // redundant with RSS link
    SKIP_DETAILS.add("title"); // redundant with RSS title
  }

  public void init(ServletConfig conf) throws ServletException {
    super.init(conf);
  }

  public void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException {

    NutchBean.LOG.info("query request from " + request.getRemoteAddr());

    // get parameters from request
    request.setCharacterEncoding("UTF-8");
    String queryString = request.getParameter("query");
    if (queryString == null) {
      queryString = "";
    }
    String urlQuery = URLEncoder.encode(queryString, "UTF-8");

    int start = 0; // first hit to display
    String startString = request.getParameter("start");
    if (startString != null) {
      start = Integer.parseInt(startString);
    }

    int hitsPerPage = DEFAULT_NUMBER_OF_HITS; // number of hits to display
    String hitsString = request.getParameter("hitsPerPage");
    if (hitsString != null) {
      hitsPerPage = Integer.parseInt(hitsString);
    }

    String sort = request.getParameter("sort");
    boolean reverse = sort != null
        && "true".equals(request.getParameter("reverse"));

    // De-Duplicate handling. Look for duplicates field and for how many
    // duplicates per results to return. Default duplicates field is 'site'
    // and duplicates per results default is '2'.
    String dedupField = request.getParameter("dedupField");
    if (dedupField == null || dedupField.length() == 0) {
      dedupField = "site";
    }
    int hitsPerDup = 2;
    String hitsPerDupString = request.getParameter("hitsPerDup");
    if (hitsPerDupString != null && hitsPerDupString.length() > 0) {
      hitsPerDup = Integer.parseInt(hitsPerDupString);
    } else {
      // If 'hitsPerSite' present, use that value.
      String hitsPerSiteString = request.getParameter("hitsPerSite");
      if (hitsPerSiteString != null && hitsPerSiteString.length() > 0) {
        hitsPerDup = Integer.parseInt(hitsPerSiteString);
      }
    }

    // Make up query string for use later drawing the 'rss' logo.
    String params = "&hitsPerPage="
        + hitsPerPage
        + (sort == null ? "" : "&sort=" + sort
            + (reverse ? "&reverse=true" : "")
            + (dedupField == null ? "" : "&dedupField=" + dedupField));

    Query query = Query.parse(queryString, getServiceLocator()
        .getConfiguration());
    NutchBean.LOG.info("query: " + queryString);

    // execute the query
    Hits hits;
    try {
      hits = getServiceLocator().getNutchBean().search(query,
          start + hitsPerPage, hitsPerDup, dedupField, sort, reverse);
    } catch (IOException e) {
      NutchBean.LOG.warn("Search Error", e);
      hits = new Hits(0, new Hit[0]);
    }

    NutchBean.LOG.info("total hits: " + hits.getTotal());

    // generate xml results
    int end = (int) Math.min(hits.getLength(), start + hitsPerPage);
    int length = end - start;

    Hit[] show = hits.getHits(start, end - start);
    HitDetails[] details = getServiceLocator().getNutchBean().getDetails(show);
    Summary[] summaries = getServiceLocator().getNutchBean().getSummary(
        details, query);

    String requestUrl = request.getRequestURL().toString();
    String base = requestUrl.substring(0, requestUrl.lastIndexOf('/'));

    try {
      DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
      factory.setNamespaceAware(true);
      Document doc = factory.newDocumentBuilder().newDocument();

      Element rss = addNode(doc, doc, "rss");
      addAttribute(doc, rss, "version", "2.0");
      addAttribute(doc, rss, "xmlns:opensearch", (String) NS_MAP
          .get("opensearch"));
      addAttribute(doc, rss, "xmlns:nutch", (String) NS_MAP.get("nutch"));

      Element channel = addNode(doc, rss, "channel");

      addNode(doc, channel, "title", "Nutch: " + queryString);
      addNode(doc, channel, "description", "Nutch search results for query: "
          + queryString);
      addNode(doc, channel, "link", base + "/search.jsp" + "?query=" + urlQuery
          + "&start=" + start + "&hitsPerDup=" + hitsPerDup + params);

      addNode(doc, channel, "opensearch", "totalResults", "" + hits.getTotal());
      addNode(doc, channel, "opensearch", "startIndex", "" + start);
      addNode(doc, channel, "opensearch", "itemsPerPage", "" + hitsPerPage);

      addNode(doc, channel, "nutch", "query", queryString);

      if ((hits.totalIsExact() && end < hits.getTotal()) // more hits to show
          || (!hits.totalIsExact() && (hits.getLength() > start + hitsPerPage))) {
        addNode(doc, channel, "nutch", "nextPage", requestUrl + "?query="
            + urlQuery + "&start=" + end + "&hitsPerDup=" + hitsPerDup + params);
      }

      if ((!hits.totalIsExact() && (hits.getLength() <= start + hitsPerPage))) {
        addNode(doc, channel, "nutch", "showAllHits", requestUrl + "?query="
            + urlQuery + "&hitsPerDup=" + 0 + params);
      }

      for (int i = 0; i < length; i++) {
        Hit hit = show[i];
        HitDetails detail = details[i];
        String title = detail.getValue("title");
        String url = detail.getValue("url");
        String id = "idx=" + hit.getIndexNo() + "&id=" + hit.getIndexDocNo();

        if (title == null || title.equals("")) {
          // use url for docs w/o title
          title = url;
        }

        Element item = addNode(doc, channel, "item");

        addNode(doc, item, "title", title);
        addNode(doc, item, "description", summaries[i].toString());
        addNode(doc, item, "link", url);

        addNode(doc, item, "nutch", "site", hit.getDedupValue());

        addNode(doc, item, "nutch", "cache", base + "/cached.do?" + id);
        addNode(doc, item, "nutch", "explain", base + "/explain.do?" + id
            + "&query=" + urlQuery);

        if (hit.moreFromDupExcluded()) {
          addNode(doc, item, "nutch", "moreFromSite", requestUrl
              + "?query="
              + URLEncoder.encode("site:" + hit.getDedupValue() + " "
                  + queryString, "UTF-8") + "&hitsPerSite=" + 0 + params);
        }

        for (int j = 0; j < detail.getLength(); j++) { // add all from detail
          String field = detail.getField(j);
          if (!SKIP_DETAILS.contains(field)) {
            addNode(doc, item, "nutch", field, detail.getValue(j));
          }
        }
      }

      // dump DOM tree

      DOMSource source = new DOMSource(doc);
      TransformerFactory transFactory = TransformerFactory.newInstance();
      Transformer transformer = transFactory.newTransformer();
      transformer.setOutputProperty("indent", "yes");
      StreamResult result = new StreamResult(response.getOutputStream());
      response.setContentType("text/xml");
      transformer.transform(source, result);

    } catch (javax.xml.parsers.ParserConfigurationException e) {
      throw new ServletException(e);
    } catch (javax.xml.transform.TransformerException e) {
      throw new ServletException(e);
    }

  }

  private static Element addNode(Document doc, Node parent, String name) {
    Element child = doc.createElement(name);
    parent.appendChild(child);
    return child;
  }

  private static void addNode(Document doc, Node parent, String name,
      String text) {
    Element child = doc.createElement(name);
    child.appendChild(doc.createTextNode(text));
    parent.appendChild(child);
  }

  private static void addNode(Document doc, Node parent, String ns,
      String name, String text) {
    Element child = doc.createElementNS((String) NS_MAP.get(ns), ns + ":"
        + name);
    child.appendChild(doc.createTextNode(text));
    parent.appendChild(child);
  }

  private static void addAttribute(Document doc, Element node, String name,
      String value) {
    Attr attribute = doc.createAttribute(name);
    attribute.setValue(value);
    node.getAttributes().setNamedItem(attribute);
  }

}