ScoringFilter.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nutch.scoring;

import java.util.Collection;
import java.util.List;
import java.util.Map.Entry;

import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.plugin.Pluggable;
import org.apache.nutch.protocol.Content;

/**
 * A contract defining behavior of scoring plugins.
 * 
 * A scoring filter will manipulate scoring variables in CrawlDatum and
 * in resulting search indexes. Filters can be chained in a specific order,
 * to provide multi-stage scoring adjustments.
 * 
 * @author Andrzej Bialecki
 */
public interface ScoringFilter extends Configurable, Pluggable {
  /** The name of the extension point. */
  public final static String X_POINT_ID = ScoringFilter.class.getName();
  
  /**
   * Set an initial score for newly injected pages. Note: newly injected pages
   * may have no inlinks, so filter implementations may wish to set this 
   * score to a non-zero value, to give newly injected pages some initial
   * credit.
   * @param url url of the page
   * @param datum new datum. Filters will modify it in-place.
   * @throws ScoringFilterException
   */
  public void injectedScore(Text url, CrawlDatum datum) throws ScoringFilterException;
  
  /**
   * Set an initial score for newly discovered pages. Note: newly discovered pages
   * have at least one inlink with its score contribution, so filter implementations
   * may choose to set initial score to zero (unknown value), and then the inlink
   * score contribution will set the "real" value of the new page.
   * @param url url of the page
   * @param datum new datum. Filters will modify it in-place.
   * @throws ScoringFilterException
   */
  public void initialScore(Text url, CrawlDatum datum) throws ScoringFilterException;
  
  /**
   * This method prepares a sort value for the purpose of sorting and
   * selecting top N scoring pages during fetchlist generation.
   * @param url url of the page
   * @param datum page's datum, should not be modified
   * @param initSort initial sort value, or a value from previous filters in chain
   */
  public float generatorSortValue(Text url, CrawlDatum datum, float initSort) throws ScoringFilterException;
  
  /**
   * This method takes all relevant score information from the current datum
   * (coming from a generated fetchlist) and stores it into
   * {@link org.apache.nutch.protocol.Content} metadata.
   * This is needed in order to pass this value(s) to the mechanism that distributes it
   * to outlinked pages.
   * @param url url of the page
   * @param datum source datum. NOTE: modifications to this value are not persisted.
   * @param content instance of content. Implementations may modify this
   * in-place, primarily by setting some metadata properties.
   */
  public void passScoreBeforeParsing(Text url, CrawlDatum datum, Content content) throws ScoringFilterException;
  
  /**
   * Currently a part of score distribution is performed using only data coming
   * from the parsing process. We need this method in order to ensure the
   * presence of score data in these steps.
   * @param url page url
   * @param content original content. NOTE: modifications to this value are not persisted.
   * @param parse target instance to copy the score information to. Implementations
   * may modify this in-place, primarily by setting some metadata properties.
   */
  public void passScoreAfterParsing(Text url, Content content, Parse parse) throws ScoringFilterException;
  
  /**
   * Distribute score value from the current page to all its outlinked pages.
   * @param fromUrl url of the source page
   * @param parseData ParseData instance, which stores relevant score value(s)
   * in its metadata. NOTE: filters may modify this in-place, all changes will
   * be persisted.
   * @param targets <url, CrawlDatum> pairs. NOTE: filters can modify this in-place,
   * all changes will be persisted.
   * @param adjust a CrawlDatum instance, initially null, which implementations
   * may use to pass adjustment values to the original CrawlDatum. When creating
   * this instance, set its status to {@link CrawlDatum#STATUS_LINKED}.
   * @param allCount number of all collected outlinks from the source page
   * @return if needed, implementations may return an instance of CrawlDatum,
   * with status {@link CrawlDatum#STATUS_LINKED}, which contains adjustments
   * to be applied to the original CrawlDatum score(s) and metadata. This can
   * be null if not needed.
   * @throws ScoringFilterException
   */
  public CrawlDatum distributeScoreToOutlinks(Text fromUrl, ParseData parseData, 
          Collection<Entry<Text, CrawlDatum>> targets, CrawlDatum adjust,
          int allCount) throws ScoringFilterException;

  /**
   * This method calculates a new score of CrawlDatum during CrawlDb update, based on the
   * initial value of the original CrawlDatum, and also score values contributed by
   * inlinked pages.
   * @param url url of the page
   * @param old original datum, with original score. May be null if this is a newly
   * discovered page. If not null, filters should use score values from this parameter
   * as the starting values - the <code>datum</code> parameter may contain values that are
   * no longer valid, if other updates occured between generation and this update.
   * @param datum the new datum, with the original score saved at the time when
   * fetchlist was generated. Filters should update this in-place, and it will be saved in
   * the crawldb.
   * @param inlinked (partial) list of CrawlDatum-s (with their scores) from
   * links pointing to this page, found in the current update batch.
   * @throws ScoringFilterException
   */
  public void updateDbScore(Text url, CrawlDatum old, CrawlDatum datum, List<CrawlDatum> inlinked) throws ScoringFilterException;
  
  /**
   * This method calculates a Lucene document boost.
   * @param url url of the page
   * @param doc Lucene document. NOTE: this already contains all information collected
   * by indexing filters. Implementations may modify this instance, in order to store/remove
   * some information.
   * @param dbDatum current page from CrawlDb. NOTE: changes made to this instance
   * are not persisted.
   * @param fetchDatum datum from FetcherOutput (containing among others the fetching status)
   * @param parse parsing result. NOTE: changes made to this instance are not persisted.
   * @param inlinks current inlinks from LinkDb. NOTE: changes made to this instance are
   * not persisted.
   * @param initScore initial boost value for the Lucene document.
   * @return boost value for the Lucene document. This value is passed as an argument
   * to the next scoring filter in chain. NOTE: implementations may also express
   * other scoring strategies by modifying Lucene document directly.
   * @throws ScoringFilterException
   */
  public float indexerScore(Text url, NutchDocument doc, CrawlDatum dbDatum,
          CrawlDatum fetchDatum, Parse parse, Inlinks inlinks, float initScore) throws ScoringFilterException;
}