/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.util;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.regex.Matcher;
import org.apache.commons.httpclient.URIException;
import org.archive.url.UsableURIFactory;
/**
* Sort-friendly URI Reordering Transform.
*
* Converts URIs of the form:
*
* scheme://userinfo@domain.tld:port/path?query#fragment
*
* ...into...
*
* scheme://(tld,domain,:port@userinfo)/path?query#fragment
*
* The '(' ')' characters serve as an unambiguous notice that the so-called
* 'authority' portion of the URI ([userinfo@]host[:port] in http URIs) has
* been transformed; the commas prevent confusion with regular hostnames.
*
* This remedies the 'problem' with standard URIs that the host portion of a
* regular URI, with its dotted-domains, is actually in reverse order from
* the natural hierarchy that's usually helpful for grouping and sorting.
*
* The value of respecting URI case variance is considered negligible: it
* is vanishingly rare for case-variance to be meaningful, while URI case-
* variance often arises from people's confusion or sloppiness, and they
* only correct it insofar as necessary to avoid blatant problems. Thus
* the usual SURT form is considered to be flattened to all lowercase, and
* not completely reversible.
*
* @author gojomo
*/
public class SURT {
protected static char DOT = '.';
protected static String BEGIN_TRANSFORMED_AUTHORITY = "(";
protected static String TRANSFORMED_HOST_DELIM = ",";
protected static String END_TRANSFORMED_AUTHORITY = ")";
// 1: scheme://
// 2: userinfo (if present)
// 3: @ (if present)
// 4: dotted-quad host
// 5: other host
// 6: :port
// 7: path
protected static String URI_SPLITTER =
"^(\\w+://)(?:([-\\w\\.!~\\*'\\(\\)%;:&=+$,]+?)(@))?"+
// 1 2 3
"(?:((?:\\d{1,3}\\.){3}\\d{1,3})|(\\S+?))(:\\d+)?(/\\S*)?$";
// 4 5 6 7
// RFC2396
// reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |
// "$" | ","
// unreserved = alphanum | mark
// mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
// userinfo = *( unreserved | escaped |
// ";" | ":" | "&" | "=" | "+" | "$" | "," )
// escaped = "%" hex hex
/**
* Utility method for creating the SURT form of the URI in the
* given String.
*
* By default, does not preserve casing.
*
* @param s String URI to be converted to SURT form
* @return SURT form
*/
public static String fromURI(String s) {
return fromURI(s,false);
}
/**
* Utility method for creating the SURT form of the URI in the
* given String.
*
* If it appears a bit convoluted in its approach, note that it was
* optimized to minimize object-creation after allocation-sites profiling
* indicated this method was a top source of garbage in long-running crawls.
*
* Assumes that the String URI has already been cleaned/fixed (eg
* by UURI fixup) in ways that put it in its crawlable form for
* evaluation.
*
* @param s String URI to be converted to SURT form
* @param preserveCase whether original case should be preserved
* @return SURT form
*/
public static String fromURI(String s, boolean preserveCase) {
Matcher m = TextUtils.getMatcher(URI_SPLITTER,s);
if(!m.matches()) {
// not an authority-based URI scheme; return unchanged
TextUtils.recycleMatcher(m);
return s;
}
// preallocate enough space for SURT form, which includes
// 3 extra characters ('(', ')', and one more ',' than '.'s
// in original)
StringBuffer builder = new StringBuffer(s.length()+3);
append(builder,s,m.start(1),m.end(1)); // scheme://
builder.append(BEGIN_TRANSFORMED_AUTHORITY); // '('
if(m.start(4)>-1) {
// dotted-quad ip match: don't reverse
append(builder,s,m.start(4),m.end(4));
} else {
// other hostname match: do reverse
int hostSegEnd = m.end(5);
int hostStart = m.start(5);
for(int i = m.end(5)-1; i>=hostStart; i--) {
if(s.charAt(i-1)!=DOT && i > hostStart) {
continue;
}
append(builder,s,i,hostSegEnd); // rev host segment
builder.append(TRANSFORMED_HOST_DELIM); // ','
hostSegEnd = i-1;
}
}
append(builder,s,m.start(6),m.end(6)); // :port
append(builder,s,m.start(3),m.end(3)); // at
append(builder,s,m.start(2),m.end(2)); // userinfo
builder.append(END_TRANSFORMED_AUTHORITY); // ')'
append(builder,s,m.start(7),m.end(7)); // path
if (!preserveCase) {
for(int i = 0; i < builder.length(); i++) {
builder.setCharAt(i,Character.toLowerCase(builder.charAt((i))));
}
}
TextUtils.recycleMatcher(m);
return builder.toString();
}
private static void append(StringBuffer b, CharSequence cs, int start,
int end) {
if (start < 0) {
return;
}
b.append(cs, start, end);
}
/**
* Given a plain URI or hostname/hostname+path, deduce an implied SURT
* prefix from it. Results may be unpredictable on strings that cannot
* be interpreted as URIs.
*
* UURI 'fixup' is applied to the URI that is built.
*
* @param u URI or almost-URI to consider
* @return implied SURT prefix form
*/
public static String prefixFromPlain(String u) {
u = fromPlain(u);
// truncate to implied prefix
u = SurtPrefixSet.asPrefix(u);
return u;
}
/**
* Given a plain URI or hostname/hostname+path, give its SURT form.
* Results may be unpredictable on strings that cannot
* be interpreted as URIs.
*
* UURI 'fixup' is applied to the URI before conversion to SURT
* form.
*
* @param u URI or almost-URI to consider
* @return implied SURT prefix form
*/
public static String fromPlain(String u) {
u = ArchiveUtils.addImpliedHttpIfNecessary(u);
boolean trailingSlash = u.endsWith("/");
// ensure all typical UURI cleanup (incl. IDN-punycoding) is done
try {
u = UsableURIFactory.getInstance(u).toString();
} catch (URIException e) {
e.printStackTrace();
// allow to continue with original string uri
}
// except: don't let UURI-fixup add a trailing slash
// if it wasn't already there (presence or absence of
// such slash has special meaning specifying implied
// SURT prefixes)
if(!trailingSlash && u.endsWith("/")) {
u = u.substring(0,u.length()-1);
}
// convert to full SURT
u = SURT.fromURI(u);
return u;
}
/**
* Allow class to be used as a command-line tool for converting
* URL lists (or naked host or host/path fragments implied
* to be HTTP URLs) to SURT form. Lines that cannot be converted
* are returned unchanged.
*
*
* Read from stdin or first file argument. Writes to stdout or
* second argument filename
*
* @param args cmd-line arguments
* @throws IOException
*/
public static void main(String[] args) throws IOException {
InputStream in = args.length > 0 ? new BufferedInputStream(
new FileInputStream(args[0])) : System.in;
PrintStream out = args.length > 1 ? new PrintStream(
new BufferedOutputStream(new FileOutputStream(args[1])))
: System.out;
BufferedReader br =
new BufferedReader(new InputStreamReader(in));
String line;
while((line = br.readLine())!=null) {
if(line.indexOf("#")>0) line=line.substring(0,line.indexOf("#"));
line = line.trim();
if(line.length()==0) continue;
line = ArchiveUtils.addImpliedHttpIfNecessary(line);
out.println(SURT.fromURI(line));
}
br.close();
out.close();
}
}