/*
* free (adj.): unencumbered; not under the control of others
* Written by mihi in 2004 and released into the public domain
* with no warranty of any kind, either expressed or implied.
* It probably won't make your computer catch on fire, or eat
* your children, but it might. Use at your own risk.
*/
package net.i2p.router.naming;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeMap;
import net.i2p.I2PAppContext;
import net.i2p.client.naming.DummyNamingService;
import net.i2p.client.naming.HostsTxtNamingService;
import net.i2p.client.naming.NamingService;
import net.i2p.client.naming.NamingServiceListener;
import net.i2p.client.naming.SingleFileNamingService;
import net.i2p.crypto.SigType;
import net.i2p.data.DataFormatException;
import net.i2p.data.DataHelper;
import net.i2p.data.Destination;
import net.i2p.data.Hash;
import net.i2p.util.LHMCache;
import net.i2p.util.Log;
import net.i2p.util.SecureFileOutputStream;
import net.i2p.util.SystemVersion;
import net.i2p.util.VersionComparator;
import net.metanotion.io.RAIFile;
import net.metanotion.io.Serializer;
import net.metanotion.io.block.BlockFile;
import net.metanotion.io.data.IntBytes;
import net.metanotion.io.data.UTF8StringBytes;
import net.metanotion.util.skiplist.SkipIterator;
import net.metanotion.util.skiplist.SkipList;
/**
* A naming service using the net.metanotion BlockFile database.
*
* This database contains the following skiplists:
* <pre>
*
* "%%__INFO__%%" is the master database skiplist, containing one entry:
* "info": a Properties, serialized with DataHelper functions:
* "version": "4"
* "created": Java long time (ms)
* "upgraded": Java long time (ms) (as of database version 2)
* "lists": Comma-separated list of host databases, to be
* searched in-order for lookups
*
* "%%__REVERSE__%%" is the reverse lookup skiplist
* (as of database version 2):
* The skiplist keys are Integers, the first 4 bytes of the hash of the dest.
* The skiplist values are Properties.
* There may be multiple entries in the properties, each one is a reverse mapping,
* as there may be more than one hostname for a given destination,
* or there could be collisions with the same first 4 bytes of the hash.
* Each property key is a hostname.
* Each property value is the empty string.
*
* For each host database, there is a skiplist containing
* the hosts for that database.
* The keys/values in these skiplists are as follows:
* key: a UTF-8 String
* value: a DestEntry, which is:
* a one-byte count of the Properties/Destination pairs to follow
* (as of database version 4, otherwise one)
* that many pairs of:
* Properties (serialized with DataHelper)
* Destination (serialized as usual).
*
*
* The DestEntry Properties typically contains:
* "a": The time added (Java long time in ms)
* "s": The original source of the entry (typically a file name or subscription URL)
* others TBD
*
* </pre>
*
* All host names are converted to lower case.
*
* @since 0.8.7, moved from core to addressbook in 0.9.31
*/
public class BlockfileNamingService extends DummyNamingService {
private final BlockFile _bf;
private final RAIFile _raf;
private final List<String> _lists;
private final List<InvalidEntry> _invalid;
private final Map<String, String> _negativeCache;
private volatile boolean _isClosed;
private final boolean _readOnly;
private String _version = "0";
private volatile boolean _isVersion4;
private boolean _needsUpgrade;
private static final Serializer<Properties> _infoSerializer = new PropertiesSerializer();
private static final Serializer<String> _stringSerializer = new UTF8StringBytes();
private static final Serializer<DestEntry> _destSerializerV1 = new DestEntrySerializer();
private static final Serializer<DestEntry> _destSerializerV4 = new DestEntrySerializerV4();
// upgrade(), initExisting(), and initNew() will change this to _destSerializerV4
private volatile Serializer<DestEntry> _destSerializer = _destSerializerV1;
private static final Serializer<Integer> _hashIndexSerializer = new IntBytes();
private static final String HOSTS_DB = "hostsdb.blockfile";
private static final String FALLBACK_LIST = "hosts.txt";
private static final String PROP_FORCE = "i2p.naming.blockfile.writeInAppContext";
private static final String INFO_SKIPLIST = "%%__INFO__%%";
private static final String REVERSE_SKIPLIST = "%%__REVERSE__%%";
private static final String PROP_INFO = "info";
private static final String PROP_VERSION = "version";
private static final String PROP_LISTVERSION = "listversion";
private static final String PROP_LISTS = "lists";
private static final String PROP_CREATED = "created";
private static final String PROP_UPGRADED = "upgraded";
private static final String VERSION = "4";
private static final String PROP_ADDED = "a";
private static final String PROP_MODDED = "m";
private static final String PROP_SOURCE = "s";
private static final String PROP_VALIDATED = "v";
private static final String DUMMY = "";
private static final int NEGATIVE_CACHE_SIZE = 32;
private static final int MAX_VALUE_LENGTH = 4096;
private static final int MAX_DESTS_PER_HOST = 8;
/**
* Opens the database at hostsdb.blockfile or creates a new
* one and imports entries from hosts.txt, userhosts.txt, and privatehosts.txt.
*
* If not in router context, the database will be opened read-only
* unless the property i2p.naming.blockfile.writeInAppContext is true.
* Not designed for multiple instantiations or simultaneous use by multple JVMs.
*
* @throws RuntimeException on fatal error
*/
public BlockfileNamingService(I2PAppContext context) {
super(context);
_lists = new ArrayList<String>();
_invalid = new ArrayList<InvalidEntry>();
_negativeCache = new LHMCache<String, String>(NEGATIVE_CACHE_SIZE);
BlockFile bf = null;
RAIFile raf = null;
boolean readOnly = false;
File f = new File(_context.getRouterDir(), HOSTS_DB);
if (f.exists()) {
try {
// closing a BlockFile does not close the underlying file,
// so we must create and retain a RAF so we may close it later
// *** Open readonly if not in router context (unless forced)
readOnly = (!f.canWrite()) ||
((!context.isRouterContext()) && (!context.getBooleanProperty(PROP_FORCE)));
raf = new RAIFile(f, true, !readOnly);
bf = initExisting(raf);
if (readOnly && context.isRouterContext())
_log.logAlways(Log.WARN, "Read-only hosts database in router context");
if (bf.wasMounted()) {
if (context.isRouterContext())
_log.logAlways(Log.WARN, "The hosts database was not closed cleanly or is still open by another process");
else
_log.logAlways(Log.WARN, "The hosts database is possibly in use by another process, perhaps the router? " +
"The database is not designed for simultaneous access by multiple processes.\n" +
"If you are using clients outside the router JVM, consider using the hosts.txt " +
"naming service with " +
"i2p.naming.impl=net.i2p.client.naming.HostsTxtNamingService");
}
} catch (IOException ioe) {
if (raf != null) {
try { raf.close(); } catch (IOException e) {}
}
File corrupt = new File(_context.getRouterDir(), HOSTS_DB + '.' + System.currentTimeMillis() + ".corrupt");
_log.log(Log.CRIT, "Corrupt, unsupported version, or unreadable database " +
f + ", moving to " + corrupt +
" and creating new database", ioe);
boolean success = f.renameTo(corrupt);
if (!success)
_log.log(Log.CRIT, "Failed to move corrupt database " + f + " to " + corrupt);
}
}
if (bf == null) {
try {
// closing a BlockFile does not close the underlying file,
// so we must create and retain a RAF so we may close it later
raf = new RAIFile(f, true, true);
SecureFileOutputStream.setPerms(f);
bf = initNew(raf);
} catch (IOException ioe) {
if (raf != null) {
try { raf.close(); } catch (IOException e) {}
}
_log.log(Log.CRIT, "Failed to initialize database", ioe);
throw new RuntimeException(ioe);
}
readOnly = false;
}
_bf = bf;
_raf = raf;
_readOnly = readOnly;
if (_needsUpgrade)
upgrade();
_context.addShutdownTask(new Shutdown());
}
/**
* Create a new database and initialize it from the local files
* privatehosts.txt, userhosts.txt, and hosts.txt,
* creating a skiplist in the database for each.
*/
private BlockFile initNew(RAIFile f) throws IOException {
long start = _context.clock().now();
_version = VERSION;
_destSerializer = _destSerializerV4;
_isVersion4 = true;
try {
BlockFile rv = new BlockFile(f, true);
SkipList<String, Properties> hdr = rv.makeIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer);
Properties info = new Properties();
info.setProperty(PROP_VERSION, VERSION);
info.setProperty(PROP_CREATED, Long.toString(_context.clock().now()));
String list = _context.getProperty(HostsTxtNamingService.PROP_HOSTS_FILE,
HostsTxtNamingService.DEFAULT_HOSTS_FILE);
info.setProperty(PROP_LISTS, list);
hdr.put(PROP_INFO, info);
rv.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
int total = 0;
for (String hostsfile : getFilenames(list)) {
File file = new File(_context.getRouterDir(), hostsfile);
if ((!file.exists()) || !(file.canRead()))
continue;
int count = 0;
BufferedReader in = null;
String sourceMsg = "Imported from " + hostsfile + " file";
try {
in = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"), 16*1024);
String line = null;
while ( (line = in.readLine()) != null) {
if (line.startsWith("#"))
continue;
int split = line.indexOf('=');
if (split <= 0)
continue;
String key = line.substring(0, split).toLowerCase(Locale.US);
if (line.indexOf('#') > 0) { // trim off any end of line comment
line = line.substring(0, line.indexOf('#')).trim();
if (line.length() < split + 1)
continue;
}
String b64 = line.substring(split+1).trim();
Destination d = lookupBase64(b64);
if (d != null) {
addEntry(rv, hostsfile, key, d, sourceMsg);
addReverseEntry(rv, key, d, _log);
count++;
} else {
_log.logAlways(Log.WARN, "Unable to import entry for " + key +
" from file " + file + " - bad Base 64: " + b64);
}
}
} catch (IOException ioe) {
_log.error("Failed to read hosts from " + file, ioe);
} finally {
if (in != null) try { in.close(); } catch (IOException ioe) {}
}
total += count;
_log.logAlways(Log.INFO, "Migrating " + count + " hosts from " + file + " to new hosts database");
_lists.add(hostsfile);
}
if (_log.shouldLog(Log.INFO))
_log.info("DB init took " + DataHelper.formatDuration(_context.clock().now() - start));
if (total <= 0)
_log.logAlways(Log.WARN, "No hosts.txt files found, Initialized hosts database with zero entries");
return rv;
} catch (RuntimeException e) {
_log.error("Failed to initialize database", e);
throw new IOException(e.toString());
}
}
/**
* Read the info block of an existing database.
*/
private BlockFile initExisting(RAIFile raf) throws IOException {
long start = _context.clock().now();
try {
BlockFile bf = new BlockFile(raf, false);
// TODO all in one skiplist or separate?
SkipList<String, Properties> hdr = bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer);
if (hdr == null)
throw new IOException("No db header");
Properties info = hdr.get(PROP_INFO);
if (info == null)
throw new IOException("No header info");
String list = info.getProperty(PROP_LISTS);
if (list == null)
throw new IOException("No lists");
long createdOn = 0;
String created = info.getProperty(PROP_CREATED);
if (created != null) {
try {
createdOn = Long.parseLong(created);
} catch (NumberFormatException nfe) {}
}
String version = info.getProperty(PROP_VERSION);
if (version == null)
throw new IOException("No version");
if (VersionComparator.comp(version, VERSION) > 0)
throw new IOException("Database version is " + version +
" but this implementation only supports versions 1-" + VERSION +
" Did you downgrade I2P??");
_version = version;
if (VersionComparator.comp(version, "4") >= 0) {
_destSerializer = _destSerializerV4;
_isVersion4 = true;
}
_needsUpgrade = needsUpgrade(bf);
if (_needsUpgrade) {
if (_log.shouldLog(Log.WARN))
_log.warn("Upgrading database from version " + _version + " to " + VERSION +
", created " + (new Date(createdOn)).toString() +
" containing lists: " + list);
} else {
if (_log.shouldLog(Log.INFO))
_log.info("Found database version " + _version +
" created " + (new Date(createdOn)).toString() +
" containing lists: " + list);
}
List<String> skiplists = getFilenames(list);
if (skiplists.isEmpty())
skiplists.add(FALLBACK_LIST);
_lists.addAll(skiplists);
if (_log.shouldLog(Log.INFO))
_log.info("DB init took " + DataHelper.formatDuration(_context.clock().now() - start));
return bf;
} catch (RuntimeException e) {
_log.error("Failed to initialize database", e);
throw new IOException(e.toString());
}
}
/**
* @return true if needs an upgrade
* @throws IOE on bad version
* @since 0.8.9
*/
private boolean needsUpgrade(BlockFile bf) throws IOException {
if (VersionComparator.comp(_version, VERSION) >= 0)
return false;
if (!bf.file.canWrite()) {
_log.logAlways(Log.WARN, "Not upgrading read-only database version " + _version);
return false;
}
return true;
}
/**
* Blockfile must be writable of course.
*
* Version 1->2: Add reverse skiplist and populate
* Version 2->3: Re-populate reverse skiplist as version 2 didn't keep it updated
* after the upgrade. No change to format.
* Version 3->4: Change format to support multiple destinations per hostname
*
* @return true if upgraded successfully
* @since 0.8.9
*/
private boolean upgrade() {
try {
// version 1 -> version 2
// Add reverse skiplist
if (VersionComparator.comp(_version, "2") < 0) {
SkipList<Integer, Properties> rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null) {
rev = _bf.makeIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (_log.shouldLog(Log.WARN))
_log.warn("Created reverse index");
}
setVersion("2");
}
// version 2 -> version 3
// no change in format, just regenerate skiplist
if (VersionComparator.comp(_version, "3") < 0) {
Map<String, Destination> entries = getEntries();
int i = 0;
for (Map.Entry<String, Destination> entry : entries.entrySet()) {
addReverseEntry(entry.getKey(), entry.getValue());
i++;
}
// i may be greater than skiplist keys if there are dups
if (_log.shouldLog(Log.WARN))
_log.warn("Updated reverse index with " + i + " entries");
setVersion("3");
}
// version 3 -> version 4
// support multiple destinations per hostname
if (VersionComparator.comp(_version, "4") < 0) {
// Upgrade of 4K entry DB on RPi 2 is over 2 1/2 minutes, probably worse on Android, disable for now
if (SystemVersion.isAndroid()) {
if (_log.shouldWarn())
_log.warn("Deferring upgrade to version 4 on Android");
return true;
}
SkipList<String, Properties> hdr = _bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer);
if (hdr == null)
throw new IOException("No db header");
Properties info = hdr.get(PROP_INFO);
if (info == null)
throw new IOException("No header info");
for (String list : _lists) {
try {
// so that we can handle an aborted upgrade,
// we keep track of the version of each list
String vprop = PROP_LISTVERSION + '_' + list;
String listVersion = info.getProperty(vprop);
if (listVersion == null || VersionComparator.comp(listVersion, "4") < 0) {
if (_log.shouldWarn())
_log.warn("Upgrading " + list + " from database version 3 to 4");
_bf.reformatIndex(list, _stringSerializer, _destSerializerV1,
_stringSerializer, _destSerializerV4);
info.setProperty(vprop, "4");
hdr.put(PROP_INFO, info);
} else {
if (_log.shouldWarn())
_log.warn("Partial upgrade, " + list + " already at version " + listVersion);
}
} catch (IOException ioe) {
_log.error("Failed upgrade of list " + list + " to version 4", ioe);
}
}
_destSerializer = _destSerializerV4;
_isVersion4 = true;
setVersion("4");
}
return true;
} catch (IOException ioe) {
_log.error("Error upgrading DB", ioe);
} catch (RuntimeException e) {
_log.error("Error upgrading DB", e);
}
return false;
}
/**
* Save new version number in blockfile after upgrade.
* Blockfile must be writable, of course.
* Side effect: sets _version field
*
* Caller must synchronize
* @since 0.9.26 pulled out of upgrade()
*/
private void setVersion(String version) throws IOException {
SkipList<String, Properties> hdr = _bf.getIndex(INFO_SKIPLIST, _stringSerializer, _infoSerializer);
if (hdr == null)
throw new IOException("No db header");
Properties info = hdr.get(PROP_INFO);
if (info == null)
throw new IOException("No header info");
info.setProperty(PROP_VERSION, version);
info.setProperty(PROP_UPGRADED, Long.toString(_context.clock().now()));
hdr.put(PROP_INFO, info);
if (_log.shouldLog(Log.WARN))
_log.warn("Upgraded database from version " + _version + " to version " + version);
_version = version;
}
/**
* For either v1 or v4.
* Caller must synchronize
* @return entry or null, or throws ioe
*/
private DestEntry getEntry(String listname, String key) throws IOException {
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null)
return null;
DestEntry rv = sl.get(key);
return rv;
} catch (IOException ioe) {
_log.error("DB Lookup error", ioe);
// delete index??
throw ioe;
} catch (RuntimeException e) {
_log.error("DB Lookup error", e);
throw new IOException(e.toString());
}
}
/**
* Caller must synchronize
* @param source may be null
*/
private void addEntry(BlockFile bf, String listname, String key, Destination dest, String source) throws IOException {
try {
// catch IOE and delete index??
SkipList<String, DestEntry> sl = bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null) {
//_log.info("Making new skiplist " + listname);
sl = bf.makeIndex(listname, _stringSerializer, _destSerializer);
}
Properties props = new Properties();
props.setProperty(PROP_ADDED, Long.toString(_context.clock().now()));
if (source != null)
props.setProperty(PROP_SOURCE, source);
addEntry(sl, key, dest, props);
} catch (IOException ioe) {
_log.error("DB add error", ioe);
// delete index??
throw ioe;
} catch (RuntimeException e) {
_log.error("DB add error", e);
throw new IOException(e.toString());
}
}
/**
* Caller must synchronize
* @param source may be null
* @throws RuntimeException
*/
/****
private void addEntry(SkipList sl, String key, Destination dest, String source) {
Properties props = new Properties();
props.setProperty(PROP_ADDED, Long.toString(_context.clock().now()));
if (source != null)
props.setProperty(PROP_SOURCE, source);
addEntry(sl, key, dest, props);
}
****/
/**
* Single dest version.
* Caller must synchronize
*
* @param props may be null
* @throws RuntimeException
*/
private static void addEntry(SkipList<String, DestEntry> sl, String key, Destination dest, Properties props) {
DestEntry de = new DestEntry();
de.dest = dest;
de.props = props;
sl.put(key, de);
}
/**
* Multiple dests version.
* DB MUST be version 4.
* Caller must synchronize
*
* @param propsList may be null, or entries may be null
* @throws RuntimeException
* @since 0.9.26
*/
private static void addEntry(SkipList<String, DestEntry> sl, String key, List<Destination> dests, List<Properties> propsList) {
DestEntry de = new DestEntry();
de.destList = dests;
de.dest = dests.get(0);
de.propsList = propsList;
if (propsList != null)
de.props = propsList.get(0);
sl.put(key, de);
}
private static List<String> getFilenames(String list) {
StringTokenizer tok = new StringTokenizer(list, ",");
List<String> rv = new ArrayList<String>(tok.countTokens());
while (tok.hasMoreTokens())
rv.add(tok.nextToken());
return rv;
}
/**
* Caller must synchronize
* @return removed object or null
* @throws RuntimeException
*/
private static <V> V removeEntry(SkipList<String, V> sl, String key) {
return sl.remove(key);
}
///// Reverse index methods
/**
* Caller must synchronize.
* @return null without exception on error (logs only)
* @since 0.8.9
*/
/****
private String getReverseEntry(Destination dest) {
return getReverseEntry(dest.calculateHash());
}
****/
/**
* Caller must synchronize.
* Returns null without exception on error (logs only).
* Returns without logging if no reverse skiplist (version 1).
*
* @return all found if more than one
* @since 0.9.26 from getReverseEntry() 0.8.9
*/
private List<String> getReverseEntries(Hash hash) {
try {
SkipList<Integer, Properties> rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
return null;
Integer idx = getReverseKey(hash);
//_log.info("Get reverse " + idx + ' ' + hash);
Properties props = rev.get(idx);
if (props == null)
return null;
List<String> rv = new ArrayList<String>(props.size());
for (String key : props.stringPropertyNames()) {
// now do the forward lookup to verify (using the cache)
List<Destination> ld = lookupAll(key);
if (ld != null) {
for (Destination d : ld) {
if (d.calculateHash().equals(hash)) {
rv.add(key);
break;
}
}
}
}
if (!rv.isEmpty())
return rv;
} catch (IOException ioe) {
_log.error("DB get reverse error", ioe);
} catch (RuntimeException e) {
_log.error("DB get reverse error", e);
}
return null;
}
/**
* Caller must synchronize.
* Fails without exception on error (logs only)
* @since 0.8.9
*/
private void addReverseEntry(String key, Destination dest) {
addReverseEntry(_bf, key, dest, _log);
}
/**
* Caller must synchronize.
* Fails without exception on error (logs only).
* Returns without logging if no reverse skiplist (version 1).
*
* We store one or more hostnames for a given hash.
* The skiplist key is a signed Integer, the first 4 bytes of the dest hash.
* For convenience (since we have a serializer already) we use
* a Properties as the value, with a null string as the value for each hostname property.
* We could in the future use the property value for something.
* @since 0.8.9
*/
private static void addReverseEntry(BlockFile bf, String key, Destination dest, Log log) {
//log.info("Add reverse " + key);
try {
SkipList<Integer, Properties> rev = bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
return;
Integer idx = getReverseKey(dest);
Properties props = rev.get(idx);
if (props != null) {
if (props.getProperty(key) != null)
return;
} else {
props = new Properties();
}
props.put(key, "");
rev.put(idx, props);
} catch (IOException ioe) {
log.error("DB add reverse error", ioe);
} catch (RuntimeException e) {
log.error("DB add reverse error", e);
}
}
/**
* Caller must synchronize.
* Fails without exception on error (logs only)
* @since 0.8.9
*/
private void removeReverseEntry(String key, Destination dest) {
//_log.info("Remove reverse " + key);
try {
SkipList<Integer, Properties> rev = _bf.getIndex(REVERSE_SKIPLIST, _hashIndexSerializer, _infoSerializer);
if (rev == null)
return;
Integer idx = getReverseKey(dest);
Properties props = rev.get(idx);
if (props == null || props.remove(key) == null)
return;
if (props.isEmpty())
rev.remove(idx);
else
rev.put(idx, props);
} catch (IOException ioe) {
_log.error("DB remove reverse error", ioe);
} catch (RuntimeException e) {
_log.error("DB remove reverse error", e);
}
}
/**
* @since 0.8.9
*/
private static Integer getReverseKey(Destination dest) {
return getReverseKey(dest.calculateHash());
}
/**
* @since 0.8.9
*/
private static Integer getReverseKey(Hash hash) {
byte[] hashBytes = hash.getData();
int i = (int) DataHelper.fromLong(hashBytes, 0, 4);
return Integer.valueOf(i);
}
////////// Start NamingService API
/*
*
* Will strip a "www." prefix and retry if lookup fails
*
* @param hostname upper/lower case ok
* @param options If non-null and contains the key "list", lookup in
* that list only, otherwise all lists
*/
@Override
public Destination lookup(String hostname, Properties lookupOptions, Properties storedOptions) {
Destination rv = lookup2(hostname, lookupOptions, storedOptions);
if (rv == null) {
// if hostname starts with "www.", strip and try again
// but not for www.i2p
hostname = hostname.toLowerCase(Locale.US);
if (hostname.startsWith("www.") && hostname.length() > 7) {
hostname = hostname.substring(4);
rv = lookup2(hostname, lookupOptions, storedOptions);
}
}
return rv;
}
/*
* Single dest version.
*
* @param lookupOptions If non-null and contains the key "list", lookup in
* that list only, otherwise all lists
*/
private Destination lookup2(String hostname, Properties lookupOptions, Properties storedOptions) {
String listname = null;
if (lookupOptions != null)
listname = lookupOptions.getProperty("list");
Destination d = null;
// only use cache if we aren't retreiving options or specifying the list
if (listname == null && storedOptions == null) {
d = super.lookup(hostname, null, null);
if (d != null)
return d;
// Base32 failed?
if (hostname.length() == BASE32_HASH_LENGTH + 8 && hostname.toLowerCase(Locale.US).endsWith(".b32.i2p"))
return null;
}
String key = hostname.toLowerCase(Locale.US);
synchronized(_negativeCache) {
if (_negativeCache.get(key) != null)
return null;
}
synchronized(_bf) {
if (_isClosed)
return null;
for (String list : _lists) {
if (listname != null && !list.equals(listname))
continue;
try {
DestEntry de = getEntry(list, key);
if (de != null) {
if (!validate(key, de, listname))
continue;
d = de.dest;
if (storedOptions != null && de.props != null)
storedOptions.putAll(de.props);
break;
}
} catch (IOException ioe) {
break;
}
}
deleteInvalid();
}
if (d != null) {
putCache(hostname, d);
} else {
synchronized(_negativeCache) {
_negativeCache.put(key, DUMMY);
}
}
return d;
}
/*
* Multiple dests version.
* DB MUST be version 4.
*
* @param lookupOptions If non-null and contains the key "list", lookup in
* that list only, otherwise all lists
* @since 0.9.26
*/
private List<Destination> lookupAll2(String hostname, Properties lookupOptions, List<Properties> storedOptions) {
// only use cache for b32
if (hostname.length() == BASE32_HASH_LENGTH + 8 && hostname.toLowerCase(Locale.US).endsWith(".b32.i2p")) {
Destination d = super.lookup(hostname, null, null);
if (d != null) {
if (storedOptions != null)
storedOptions.add(null);
return Collections.singletonList(d);
}
// Base32 failed?
return null;
}
String key = hostname.toLowerCase(Locale.US);
synchronized(_negativeCache) {
if (_negativeCache.get(key) != null)
return null;
}
String listname = null;
if (lookupOptions != null)
listname = lookupOptions.getProperty("list");
List<Destination> rv = null;
synchronized(_bf) {
if (_isClosed)
return null;
for (String list : _lists) {
if (listname != null && !list.equals(listname))
continue;
try {
DestEntry de = getEntry(list, key);
if (de != null) {
if (!validate(key, de, listname))
continue;
if (de.destList != null) {
rv = de.destList;
if (storedOptions != null)
storedOptions.addAll(de.propsList);
} else {
rv = Collections.singletonList(de.dest);
if (storedOptions != null)
storedOptions.add(de.props);
}
break;
}
} catch (IOException ioe) {
break;
}
}
deleteInvalid();
}
if (rv != null) {
putCache(hostname, rv.get(0));
} else {
synchronized(_negativeCache) {
_negativeCache.put(key, DUMMY);
}
}
return rv;
}
/**
* @param options If non-null and contains the key "list", add to that list
* (default "hosts.txt")
* Use the key "s" for the source
*/
@Override
public boolean put(String hostname, Destination d, Properties options) {
return put(hostname, d, options, false);
}
/**
* @param options If non-null and contains the key "list", add to that list
* (default "hosts.txt")
* Use the key "s" for the source.
* Key "a" will be added with the current time, unless
* "a" is present in options.
*/
@Override
public boolean putIfAbsent(String hostname, Destination d, Properties options) {
return put(hostname, d, options, true);
}
/**
* Single dest version
* This does not prevent adding b32. Caller must check.
*
* @param checkExisting if true, fail if entry already exists
*/
private boolean put(String hostname, Destination d, Properties options, boolean checkExisting) {
if (_readOnly) {
_log.error("Add entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase(Locale.US);
synchronized(_negativeCache) {
_negativeCache.remove(key);
}
String listname = FALLBACK_LIST;
Properties props = new Properties();
props.setProperty(PROP_ADDED, Long.toString(_context.clock().now()));
if (options != null) {
props.putAll(options);
String list = options.getProperty("list");
if (list != null) {
listname = list;
props.remove("list");
}
}
synchronized(_bf) {
if (_isClosed)
return false;
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null)
sl = _bf.makeIndex(listname, _stringSerializer, _destSerializer);
boolean changed = (checkExisting || !_listeners.isEmpty()) && sl.get(key) != null;
if (changed && checkExisting)
return false;
addEntry(sl, key, d, props);
if (changed) {
removeCache(hostname);
// removeReverseEntry(key, oldDest) ???
}
addReverseEntry(key, d);
for (NamingServiceListener nsl : _listeners) {
if (changed)
nsl.entryChanged(this, hostname, d, options);
else
nsl.entryAdded(this, hostname, d, options);
}
return true;
} catch (IOException ioe) {
_log.error("DB add error", ioe);
return false;
} catch (RuntimeException re) {
_log.error("DB add error", re);
return false;
}
}
}
/**
* Multiple dests version.
* DB MUST be version 4.
* This does not prevent adding b32. Caller must check.
*
* @param propsList may be null, or entries may be null
* @param checkExisting if true, fail if entry already exists
* @since 0.9.26
*/
private boolean put(String hostname, List<Destination> dests, List<Properties> propsList, boolean checkExisting) {
int sz = dests.size();
if (sz <= 0)
throw new IllegalArgumentException();
if (sz == 1)
return put(hostname, dests.get(0), propsList != null ? propsList.get(0) : null, checkExisting);
if (_readOnly) {
_log.error("Add entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase(Locale.US);
synchronized(_negativeCache) {
_negativeCache.remove(key);
}
String listname = FALLBACK_LIST;
String date = Long.toString(_context.clock().now());
List<Properties> outProps = new ArrayList<Properties>(propsList.size());
for (Properties options : propsList) {
Properties props = new Properties();
props.setProperty(PROP_ADDED, date);
if (options != null) {
props.putAll(options);
String list = options.getProperty("list");
if (list != null) {
listname = list;
props.remove("list");
}
}
outProps.add(props);
}
synchronized(_bf) {
if (_isClosed)
return false;
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null)
sl = _bf.makeIndex(listname, _stringSerializer, _destSerializer);
boolean changed = (checkExisting || !_listeners.isEmpty()) && sl.get(key) != null;
if (changed && checkExisting)
return false;
addEntry(sl, key, dests, outProps);
if (changed) {
removeCache(hostname);
// removeReverseEntry(key, oldDest) ???
}
for (int i = 0; i < dests.size(); i++) {
Destination d = dests.get(i);
Properties options = propsList.get(i);
addReverseEntry(key, d);
for (NamingServiceListener nsl : _listeners) {
if (changed)
nsl.entryChanged(this, hostname, d, options);
else
nsl.entryAdded(this, hostname, d, options);
}
}
return true;
} catch (IOException ioe) {
_log.error("DB add error", ioe);
return false;
} catch (RuntimeException re) {
_log.error("DB add error", re);
return false;
}
}
}
/**
* @param options If non-null and contains the key "list", remove
* from that list (default "hosts.txt", NOT all lists)
*/
@Override
public boolean remove(String hostname, Properties options) {
if (_readOnly) {
_log.error("Remove entry failed, read-only hosts database");
return false;
}
String key = hostname.toLowerCase(Locale.US);
String listname = FALLBACK_LIST;
if (options != null) {
String list = options.getProperty("list");
if (list != null) {
listname = list;
}
}
synchronized(_bf) {
if (_isClosed)
return false;
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null)
return false;
DestEntry removed = removeEntry(sl, key);
boolean rv = removed != null;
if (rv) {
removeCache(hostname);
try {
removeReverseEntry(key, removed.dest);
} catch (ClassCastException cce) {
_log.error("DB reverse remove error", cce);
}
for (NamingServiceListener nsl : _listeners) {
nsl.entryRemoved(this, key);
}
}
return rv;
} catch (IOException ioe) {
_log.error("DB remove error", ioe);
return false;
} catch (RuntimeException re) {
_log.error("DB remove error", re);
return false;
}
}
}
/**
* @param options If non-null and contains the key "list", get
* from that list (default "hosts.txt", NOT all lists)
* Key "skip": skip that many entries
* Key "limit": max number to return
* Key "search": return only those matching substring
* Key "startsWith": return only those starting with
* ("[0-9]" allowed)
* Key "beginWith": start here in the iteration
* Don't use both startsWith and beginWith.
* Search, startsWith, and beginWith values must be lower case.
*/
@Override
public Map<String, Destination> getEntries(Properties options) {
String listname = FALLBACK_LIST;
String search = null;
String startsWith = null;
String beginWith = null;
int limit = Integer.MAX_VALUE;
int skip = 0;
if (options != null) {
String ln = options.getProperty("list");
if (ln != null)
listname = ln;
search = options.getProperty("search");
startsWith = options.getProperty("startsWith");
beginWith = options.getProperty("beginWith");
if (beginWith == null && startsWith != null) {
if (startsWith.equals("[0-9]"))
beginWith = "0";
else
beginWith = startsWith;
}
String lim = options.getProperty("limit");
try {
limit = Integer.parseInt(lim);
} catch (NumberFormatException nfe) {}
String sk = options.getProperty("skip");
try {
skip = Integer.parseInt(sk);
} catch (NumberFormatException nfe) {}
}
if (_log.shouldLog(Log.DEBUG))
_log.debug("Searching " + listname + " beginning with " + beginWith +
" starting with " + startsWith + " search string " + search +
" limit=" + limit + " skip=" + skip);
synchronized(_bf) {
if (_isClosed)
return Collections.emptyMap();
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null) {
if (_log.shouldLog(Log.WARN))
_log.warn("No skiplist found for lookup in " + listname);
return Collections.emptyMap();
}
SkipIterator<String, DestEntry> iter;
if (beginWith != null)
iter = sl.find(beginWith);
else
iter = sl.iterator();
Map<String, Destination> rv = new TreeMap<String, Destination>();
for (int i = 0; i < skip && iter.hasNext(); i++) {
// don't bother validating here
iter.next();
}
for (int i = 0; i < limit && iter.hasNext(); ) {
String key = iter.nextKey();
if (startsWith != null) {
if (startsWith.equals("[0-9]")) {
if (key.charAt(0) > '9')
break;
} else if (!key.startsWith(startsWith)) {
break;
}
}
DestEntry de = iter.next();
if (!validate(key, de, listname))
continue;
if (search != null && key.indexOf(search) < 0)
continue;
rv.put(key, de.dest);
i++;
}
return rv;
} catch (IOException ioe) {
_log.error("DB lookup error", ioe);
return Collections.emptyMap();
} catch (RuntimeException re) {
_log.error("DB lookup error", re);
return Collections.emptyMap();
} finally {
deleteInvalid();
}
}
}
/**
* @param options If non-null and contains the key "list", get
* from that list (default "hosts.txt", NOT all lists)
* Key "skip": skip that many entries
* Key "limit": max number to return
* Key "search": return only those matching substring
* Key "startsWith": return only those starting with
* ("[0-9]" allowed)
* Key "beginWith": start here in the iteration
* Don't use both startsWith and beginWith.
* Search, startsWith, and beginWith values must be lower case.
* @since 0.9.20
*/
@Override
public Map<String, String> getBase64Entries(Properties options) {
String listname = FALLBACK_LIST;
String search = null;
String startsWith = null;
String beginWith = null;
int limit = Integer.MAX_VALUE;
int skip = 0;
if (options != null) {
String ln = options.getProperty("list");
if (ln != null)
listname = ln;
search = options.getProperty("search");
startsWith = options.getProperty("startsWith");
beginWith = options.getProperty("beginWith");
if (beginWith == null && startsWith != null) {
if (startsWith.equals("[0-9]"))
beginWith = "0";
else
beginWith = startsWith;
}
String lim = options.getProperty("limit");
try {
limit = Integer.parseInt(lim);
} catch (NumberFormatException nfe) {}
String sk = options.getProperty("skip");
try {
skip = Integer.parseInt(sk);
} catch (NumberFormatException nfe) {}
}
synchronized(_bf) {
if (_isClosed)
return Collections.emptyMap();
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null) {
if (_log.shouldLog(Log.WARN))
_log.warn("No skiplist found for lookup in " + listname);
return Collections.emptyMap();
}
SkipIterator<String, DestEntry> iter;
if (beginWith != null)
iter = sl.find(beginWith);
else
iter = sl.iterator();
Map<String, String> rv = new TreeMap<String, String>();
for (int i = 0; i < skip && iter.hasNext(); i++) {
// don't bother validating here
iter.next();
}
for (int i = 0; i < limit && iter.hasNext(); ) {
String key = iter.nextKey();
if (startsWith != null) {
if (startsWith.equals("[0-9]")) {
if (key.charAt(0) > '9')
break;
} else if (!key.startsWith(startsWith)) {
break;
}
}
DestEntry de = iter.next();
if (!validate(key, de, listname))
continue;
if (search != null && key.indexOf(search) < 0)
continue;
rv.put(key, de.dest.toBase64());
i++;
}
return rv;
} catch (IOException ioe) {
_log.error("DB lookup error", ioe);
return Collections.emptyMap();
} catch (RuntimeException re) {
_log.error("DB lookup error", re);
return Collections.emptyMap();
} finally {
deleteInvalid();
}
}
}
/**
* Export in a hosts.txt format.
* Output is sorted.
* Caller must close writer.
*
* @param options If non-null and contains the key "list", get
* from that list (default "hosts.txt", NOT all lists)
* Key "search": return only those matching substring
* Key "startsWith": return only those starting with
* ("[0-9]" allowed)
* Key "beginWith": start here in the iteration
* @since 0.9.30 override NamingService to add stored authentication strings
*/
@Override
public void export(Writer out, Properties options) throws IOException {
String listname = FALLBACK_LIST;
String search = null;
String startsWith = null;
String beginWith = null;
if (options != null) {
String ln = options.getProperty("list");
if (ln != null)
listname = ln;
search = options.getProperty("search");
startsWith = options.getProperty("startsWith");
beginWith = options.getProperty("beginWith");
if (beginWith == null && startsWith != null) {
if (startsWith.equals("[0-9]"))
beginWith = "0";
else
beginWith = startsWith;
}
}
out.write("# Address book: ");
out.write(getName());
out.write(" (" + listname + ')');
final String nl = System.getProperty("line.separator", "\n");
out.write(nl);
out.write("# Exported: ");
out.write((new Date()).toString());
out.write(nl);
synchronized(_bf) {
if (_isClosed)
return;
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null) {
if (_log.shouldLog(Log.WARN))
_log.warn("No skiplist found for lookup in " + listname);
return;
}
if (beginWith == null && search == null) {
int sz = sl.size();
if (sz <= 0) {
out.write("# No entries");
out.write(nl);
return;
}
if (sz > 1) {
// actually not right due to multidest
out.write("# " + sz + " entries");
out.write(nl);
}
}
SkipIterator<String, DestEntry> iter;
if (beginWith != null)
iter = sl.find(beginWith);
else
iter = sl.iterator();
int cnt = 0;
while (iter.hasNext()) {
String key = iter.nextKey();
if (startsWith != null) {
if (startsWith.equals("[0-9]")) {
if (key.charAt(0) > '9')
break;
} else if (!key.startsWith(startsWith)) {
break;
}
}
DestEntry de = iter.next();
if (!validate(key, de, listname))
continue;
if (search != null && key.indexOf(search) < 0)
continue;
int dsz = de.destList != null ? de.destList.size() : 1;
// new non-DSA dest is put first, so put in reverse
// order so importers will see the older dest first
for (int i = dsz - 1; i >= 0; i--) {
Properties p;
Destination d;
if (i == 0) {
p = de.props;
d = de.dest;
} else {
p = de.propsList.get(i);
d = de.destList.get(i);
}
out.write(key);
out.write('=');
out.write(d.toBase64());
if (p != null)
SingleFileNamingService.writeOptions(p, out);
out.write(nl);
cnt++;
}
}
if (beginWith != null || search != null) {
if (cnt <= 0) {
out.write("# No entries");
out.write(nl);
return;
}
if (cnt > 1) {
out.write("# " + cnt + " entries");
out.write(nl);
}
}
} catch (RuntimeException re) {
throw new IOException("DB lookup error", re);
} finally {
deleteInvalid();
}
}
}
/**
* @param options If non-null and contains the key "list", get
* from that list (default "hosts.txt", NOT all lists)
* Key "skip": skip that many entries
* Key "limit": max number to return
* Key "search": return only those matching substring
* Key "startsWith": return only those starting with
* ("[0-9]" allowed)
* Key "beginWith": start here in the iteration
* Don't use both startsWith and beginWith.
* Search, startsWith, and beginWith values must be lower case.
* @since 0.9.20
*/
@Override
public Set<String> getNames(Properties options) {
String listname = FALLBACK_LIST;
String search = null;
String startsWith = null;
String beginWith = null;
int limit = Integer.MAX_VALUE;
int skip = 0;
if (options != null) {
String ln = options.getProperty("list");
if (ln != null)
listname = ln;
search = options.getProperty("search");
startsWith = options.getProperty("startsWith");
beginWith = options.getProperty("beginWith");
if (beginWith == null && startsWith != null) {
if (startsWith.equals("[0-9]"))
beginWith = "0";
else
beginWith = startsWith;
}
String lim = options.getProperty("limit");
try {
limit = Integer.parseInt(lim);
} catch (NumberFormatException nfe) {}
String sk = options.getProperty("skip");
try {
skip = Integer.parseInt(sk);
} catch (NumberFormatException nfe) {}
}
synchronized(_bf) {
if (_isClosed)
return Collections.emptySet();
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null) {
if (_log.shouldLog(Log.WARN))
_log.warn("No skiplist found for lookup in " + listname);
return Collections.emptySet();
}
SkipIterator<String, DestEntry> iter;
if (beginWith != null)
iter = sl.find(beginWith);
else
iter = sl.iterator();
Set<String> rv = new HashSet<String>();
for (int i = 0; i < skip && iter.hasNext(); i++) {
iter.next();
}
for (int i = 0; i < limit && iter.hasNext(); ) {
String key = iter.nextKey();
if (startsWith != null) {
if (startsWith.equals("[0-9]")) {
if (key.charAt(0) > '9')
break;
} else if (!key.startsWith(startsWith)) {
break;
}
}
if (search != null && key.indexOf(search) < 0)
continue;
rv.add(key);
i++;
}
return rv;
} catch (IOException ioe) {
_log.error("DB lookup error", ioe);
return Collections.emptySet();
} catch (RuntimeException re) {
_log.error("DB lookup error", re);
return Collections.emptySet();
}
}
}
/**
* @param options ignored
* @since 0.8.9
*/
@Override
public String reverseLookup(Destination d, Properties options) {
return reverseLookup(d.calculateHash());
}
/**
* @since 0.8.9
*/
@Override
public String reverseLookup(Hash h) {
List<String> ls;
synchronized(_bf) {
if (_isClosed)
return null;
ls = getReverseEntries(h);
}
return (ls != null) ? ls.get(0) : null;
}
/**
* @param options ignored
* @since 0.9.26
*/
@Override
public List<String> reverseLookupAll(Destination d, Properties options) {
return reverseLookupAll(d.calculateHash());
}
/**
* @since 0.9.26
*/
@Override
public List<String> reverseLookupAll(Hash h) {
synchronized(_bf) {
if (_isClosed)
return null;
return getReverseEntries(h);
}
}
/**
* @param options If non-null and contains the key "list", return the
* size of that list (default "hosts.txt", NOT all lists)
*/
@Override
public int size(Properties options) {
String listname = FALLBACK_LIST;
if (options != null) {
String list = options.getProperty("list");
if (list != null) {
listname = list;
}
}
synchronized(_bf) {
if (_isClosed)
return 0;
try {
SkipList<String, DestEntry> sl = _bf.getIndex(listname, _stringSerializer, _destSerializer);
if (sl == null)
return 0;
return sl.size();
} catch (IOException ioe) {
_log.error("DB size error", ioe);
return 0;
} catch (RuntimeException re) {
_log.error("DB size error", re);
return 0;
}
}
}
public void shutdown() {
close();
}
////////// End NamingService API
//// Begin new API for multiple Destinations
/**
* Return all of the entries found in the first list found, or in the list
* specified in lookupOptions. Does not aggregate all destinations found
* in all lists.
*
* If storedOptions is non-null, it must be a List that supports null entries.
* If the returned value (the List of Destinations) is non-null,
* the same number of Properties objects will be added to storedOptions.
* If no properties were found for a given Destination, the corresponding
* entry in the storedOptions list will be null.
*
* @param lookupOptions input parameter, NamingService-specific, may be null
* @param storedOptions output parameter, NamingService-specific, any stored properties will be added if non-null
* @return non-empty List of Destinations, or null if nothing found
* @since 0.9.26
*/
@Override
public List<Destination> lookupAll(String hostname, Properties lookupOptions, List<Properties> storedOptions) {
if (!_isVersion4)
return super.lookupAll(hostname, lookupOptions, storedOptions);
List<Destination> rv = lookupAll2(hostname, lookupOptions, storedOptions);
if (rv == null) {
// if hostname starts with "www.", strip and try again
// but not for www.i2p
hostname = hostname.toLowerCase(Locale.US);
if (hostname.startsWith("www.") && hostname.length() > 7) {
hostname = hostname.substring(4);
rv = lookupAll2(hostname, lookupOptions, storedOptions);
}
}
// we sort the destinations in addDestination(),
// which is a lot easier than sorting them here
return rv;
}
/**
* Add a Destination to an existing hostname's entry in the addressbook.
*
* This does not prevent adding b32. Caller must check.
*
* @param options NamingService-specific, may be null
* @return success
* @since 0.9.26
*/
@Override
public boolean addDestination(String hostname, Destination d, Properties options) {
if (!_isVersion4)
return putIfAbsent(hostname, d, options);
List<Properties> storedOptions = new ArrayList<Properties>(4);
synchronized(_bf) {
// We use lookupAll2(), not lookupAll(), because if hostname starts with www.,
// we do not want to read in from the
// non-www hostname and then copy it to a new www hostname.
List<Destination> dests = lookupAll2(hostname, options, storedOptions);
if (dests == null)
return put(hostname, d, options, false);
if (dests.contains(d))
return false;
if (dests.size() >= MAX_DESTS_PER_HOST)
return false;
List<Destination> newDests = new ArrayList<Destination>(dests.size() + 1);
newDests.addAll(dests);
// TODO better sort by sigtype preference.
// For now, non-DSA at the front, DSA at the end
SigType type = d.getSigningPublicKey().getType();
if (type != SigType.DSA_SHA1 && type.isAvailable()) {
newDests.add(0, d);
storedOptions.add(0, options);
} else {
newDests.add(d);
storedOptions.add(options);
}
return put(hostname, newDests, storedOptions, false);
}
}
/**
* Remove a hostname's entry only if it contains the Destination d.
* If the NamingService supports multiple Destinations per hostname,
* and this is the only Destination, removes the entire entry.
* If aditional Destinations remain, it only removes the
* specified Destination from the entry.
*
* @param options NamingService-specific, may be null
* @return true if entry containing d was successfully removed.
* @since 0.9.26
*/
@Override
public boolean remove(String hostname, Destination d, Properties options) {
if (!_isVersion4) {
// super does a get-test-remove, so lock around that
synchronized(_bf) {
return super.remove(hostname, d, options);
}
}
List<Properties> storedOptions = new ArrayList<Properties>(4);
synchronized(_bf) {
// We use lookupAll2(), not lookupAll(), because if hostname starts with www.,
// we do not want to read in from the
// non-www hostname and then copy it to a new www hostname.
List<Destination> dests = lookupAll2(hostname, options, storedOptions);
if (dests == null)
return false;
for (int i = 0; i < dests.size(); i++) {
Destination dd = dests.get(i);
if (dd.equals(d)) {
// Found it. Remove and return.
if (dests.size() == 1)
return remove(hostname, options);
List<Destination> newDests = new ArrayList<Destination>(dests.size() - 1);
for (int j = 0; j < dests.size(); j++) {
if (j != i)
newDests.add(dests.get(j));
}
storedOptions.remove(i);
removeReverseEntry(hostname, d);
if (options != null) {
String list = options.getProperty("list");
if (list != null)
storedOptions.get(0).setProperty("list", list);
}
return put(hostname, newDests, storedOptions, false);
}
}
}
return false;
}
//// End new API for multiple Destinations
/**
* Continuously validate anything we read in.
* Queue anything invalid to be removed at the end of the operation.
* Caller must sync!
* @return valid
*/
private boolean validate(String key, DestEntry de, String listname) {
if (key == null)
return false;
// de.props may be null
// publickey check is a quick proxy to detect dest deserialization failure
boolean rv = key.length() > 0 &&
de != null &&
de.dest != null &&
de.dest.getPublicKey() != null;
if (_isVersion4 && rv && de.destList != null) {
// additional checks for multi-dest
rv = de.propsList != null &&
de.destList.size() == de.propsList.size() &&
!de.destList.contains(null);
}
if ((!rv) && (!_readOnly))
_invalid.add(new InvalidEntry(key, listname));
return rv;
}
/**
* Remove and log all invalid entries queued by validate()
* while scanning in lookup() or getEntries().
* We delete in the order detected, as an error may be corrupting later entries in the skiplist.
* Caller must sync!
*/
private void deleteInvalid() {
if (_invalid.isEmpty())
return;
_log.error("Removing " + _invalid.size() + " corrupt entries from database");
for (InvalidEntry ie : _invalid) {
String key = ie.key;
String list = ie.list;
try {
SkipList<String, DestEntry> sl = _bf.getIndex(list, _stringSerializer, _destSerializer);
if (sl == null) {
_log.error("No list found to remove corrupt \"" + key + "\" from database " + list);
continue;
}
// this will often return null since it was corrupt
boolean success = removeEntry(sl, key) != null;
if (success)
_log.error("Removed corrupt \"" + key + "\" from database " + list);
else
_log.error("May have Failed to remove corrupt \"" + key + "\" from database " + list);
} catch (RuntimeException re) {
_log.error("Error while removing corrupt \"" + key + "\" from database " + list, re);
} catch (IOException ioe) {
_log.error("Error while removing corrput \"" + key + "\" from database " + list, ioe);
}
}
_invalid.clear();
}
/****
private void dumpDB() {
synchronized(_bf) {
if (_isClosed)
_log.error("Database is closed");
for (String list : _lists) {
try {
SkipList sl = _bf.getIndex(list, _stringSerializer, _destSerializer);
if (sl == null) {
_log.error("No list found for " + list);
continue;
}
int i = 0;
for (SkipIterator iter = sl.iterator(); iter.hasNext(); ) {
String key = (String) iter.nextKey();
DestEntry de = (DestEntry) iter.next();
if (!validate(key, de, list))
continue;
_log.error("DB " + list + " key " + key + " val " + de);
i++;
}
_log.error(i + " entries found for " + list);
} catch (IOException ioe) {
_log.error("Fail", ioe);
break;
}
}
deleteInvalid();
}
}
****/
private void close() {
synchronized(_bf) {
try {
_bf.close();
} catch (IOException ioe) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error closing", ioe);
} catch (RuntimeException e) {
if (_log.shouldLog(Log.WARN))
_log.warn("Error closing", e);
}
try {
_raf.close();
} catch (IOException ioe) {
}
_isClosed = true;
}
synchronized(_negativeCache) {
_negativeCache.clear();
}
clearCache();
}
/** for logging errors in the static serializers below */
private static void logError(String msg, Throwable t) {
I2PAppContext.getGlobalContext().logManager().getLog(BlockfileNamingService.class).error(msg, t);
}
private class Shutdown implements Runnable {
public void run() {
close();
}
}
/**
* Used for the values in the header skiplist
* Take care not to throw on any error.
* This means that some things will fail with no indication other than the log,
* but if we threw a RuntimeException we would prevent access to entries later in
* the SkipSpan.
*/
private static class PropertiesSerializer implements Serializer<Properties> {
/**
* A format error on the properties is non-fatal (returns an empty properties)
*/
public byte[] getBytes(Properties p) {
try {
return DataHelper.toProperties(p);
} catch (DataFormatException dfe) {
logError("DB Write Fail - properties too big?", dfe);
// null properties is a two-byte length of 0.
return new byte[2];
}
}
/** returns null on error */
public Properties construct(byte[] b) {
Properties rv = new Properties();
try {
DataHelper.fromProperties(b, 0, rv);
} catch (DataFormatException dfe) {
logError("DB Read Fail", dfe);
return null;
}
return rv;
}
}
/**
* A DestEntry contains Properties and a Destination,
* and is serialized in that order.
*/
private static class DestEntry {
/** May be null.
* If more than one dest, contains the first props.
*/
public Properties props;
/** May not be null.
* If more than one dest, contains the first dest.
*/
public Destination dest;
/** May be null - v4 only - same size as destList - may contain null entries
* Only non-null if more than one dest.
* First entry always equal to props.
*/
public List<Properties> propsList;
/** May be null - v4 only - same size as propsList
* Only non-null if more than one dest.
* First entry always equal to dest.
*/
public List<Destination> destList;
@Override
public String toString() {
return "DestEntry (" + DataHelper.toString(props) +
") " + dest.toString();
}
}
/**
* Used for the values in the addressbook skiplists
* Take care not to throw on any error.
* This means that some things will fail with no indication other than the log,
* but if we threw a RuntimeException we would prevent access to entries later in
* the SkipSpan.
*/
private static class DestEntrySerializer implements Serializer<DestEntry> {
/**
* A format error on the properties is non-fatal (only the properties are lost)
* A format error on the destination is fatal
*/
public byte[] getBytes(DestEntry de) {
ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
try {
try {
DataHelper.writeProperties(baos, de.props, true, false); // UTF-8, unsorted
} catch (DataFormatException dfe) {
logError("DB Write Fail - properties too big?", dfe);
// null properties is a two-byte length of 0.
baos.write(new byte[2]);
}
de.dest.writeBytes(baos);
} catch (IOException ioe) {
logError("DB Write Fail", ioe);
} catch (DataFormatException dfe) {
logError("DB Write Fail", dfe);
}
return baos.toByteArray();
}
/** returns null on error */
public DestEntry construct(byte[] b) {
DestEntry rv = new DestEntry();
ByteArrayInputStream bais = new ByteArrayInputStream(b);
try {
rv.props = DataHelper.readProperties(bais);
//dest.readBytes(bais);
// Will this flush the dest cache too much?
rv.dest = Destination.create(bais);
} catch (IOException ioe) {
logError("DB Read Fail", ioe);
return null;
} catch (DataFormatException dfe) {
logError("DB Read Fail", dfe);
return null;
}
return rv;
}
}
/**
* For multiple destinations per hostname
* @since 0.9.26
*/
private static class DestEntrySerializerV4 implements Serializer<DestEntry> {
public byte[] getBytes(DestEntry de) {
ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
int sz = de.destList != null ? de.destList.size() : 1;
try {
baos.write((byte) sz);
for (int i = 0; i < sz; i++) {
Properties p;
Destination d;
if (i == 0) {
p = de.props;
d = de.dest;
} else {
p = de.propsList.get(i);
d = de.destList.get(i);
}
try {
writeProperties(baos, p);
} catch (DataFormatException dfe) {
logError("DB Write Fail - properties too big?", dfe);
baos.write(new byte[2]);
}
d.writeBytes(baos);
}
} catch (IOException ioe) {
logError("DB Write Fail", ioe);
} catch (DataFormatException dfe) {
logError("DB Write Fail", dfe);
}
return baos.toByteArray();
}
/** returns null on error */
public DestEntry construct(byte[] b) {
DestEntry rv = new DestEntry();
ByteArrayInputStream bais = new ByteArrayInputStream(b);
try {
int sz = bais.read() & 0xff;
if (sz <= 0)
throw new DataFormatException("bad dest count " + sz);
rv.props = readProperties(bais);
rv.dest = Destination.create(bais);
if (sz > 1) {
rv.propsList = new ArrayList<Properties>(sz);
rv.destList = new ArrayList<Destination>(sz);
rv.propsList.add(rv.props);
rv.destList.add(rv.dest);
for (int i = 1; i < sz; i++) {
rv.propsList.add(readProperties(bais));
rv.destList.add(Destination.create(bais));
}
}
} catch (IOException ioe) {
logError("DB Read Fail", ioe);
return null;
} catch (DataFormatException dfe) {
logError("DB Read Fail", dfe);
return null;
}
return rv;
}
}
/**
* Same as DataHelper.writeProperties, UTF-8, unsorted,
* except that values may up to 4K bytes.
*
* @param props source may be null
* @throws DataFormatException if any key string is over 255 bytes long,
* if any value string is over 4096 bytes long, or if the total length
* (not including the two length bytes) is greater than 65535 bytes.
* @since 0.9.26
*/
private static void writeProperties(ByteArrayOutputStream rawStream, Properties p)
throws DataFormatException, IOException {
if (p != null && !p.isEmpty()) {
ByteArrayOutputStream baos = new ByteArrayOutputStream(p.size() * 32);
for (Map.Entry<Object, Object> entry : p.entrySet()) {
String key = (String) entry.getKey();
String val = (String) entry.getValue();
DataHelper.writeStringUTF8(baos, key);
baos.write('=');
writeLongStringUTF8(baos, val);
baos.write(';');
}
if (baos.size() > 65535)
throw new DataFormatException("Properties too big (65535 max): " + baos.size());
byte propBytes[] = baos.toByteArray();
DataHelper.writeLong(rawStream, 2, propBytes.length);
rawStream.write(propBytes);
} else {
DataHelper.writeLong(rawStream, 2, 0);
}
}
/**
* Same as DataHelper.readProperties, UTF-8, unsorted,
* except that values may up to 4K bytes.
*
* Throws DataFormatException on duplicate key
*
* @param in stream to read the mapping from
* @throws DataFormatException if the format is invalid
* @throws IOException if there is a problem reading the data
* @return a Properties
* @since 0.9.26
*/
public static Properties readProperties(ByteArrayInputStream in)
throws DataFormatException, IOException {
Properties props = new Properties();
int size = (int) DataHelper.readLong(in, 2);
// this doesn't prevent reading past the end on corruption
int ignore = in.available() - size;
while (in.available() > ignore) {
String key = DataHelper.readString(in);
int b = in.read();
if (b != '=')
throw new DataFormatException("Bad key " + b);
String val = readLongString(in);
b = in.read();
if (b != ';')
throw new DataFormatException("Bad value");
Object old = props.put(key, val);
if (old != null)
throw new DataFormatException("Duplicate key " + key);
}
return props;
}
/**
* Same as DataHelper.writeStringUTF8, except that
* strings up to 4K bytes are allowed.
* Format is: one-byte length + data, or 0xff + two-byte length + data
*
* @param out stream to write string
* @param string to write out: null strings are valid, but strings of excess length will
* cause a DataFormatException to be thrown
* @throws DataFormatException if the string is not valid
* @throws IOException if there is an IO error writing the string
*/
private static void writeLongStringUTF8(ByteArrayOutputStream out, String string)
throws DataFormatException, IOException {
if (string == null) {
out.write(0);
} else {
byte[] raw = string.getBytes("UTF-8");
int len = raw.length;
if (len >= 255) {
if (len > MAX_VALUE_LENGTH)
throw new DataFormatException(MAX_VALUE_LENGTH + " max, but this is "
+ len + " [" + string + "]");
out.write(0xff);
DataHelper.writeLong(out, 2, len);
} else {
out.write(len);
}
out.write(raw);
}
}
/**
* Same as DataHelper.readString, except that
* strings up to 4K bytes are allowed.
* Format is: one-byte length + data, or 0xff + two-byte length + data
*
* @param in stream to read from
* @throws DataFormatException if the stream doesn't contain a validly formatted string
* @throws EOFException if there aren't enough bytes to read the string
* @throws IOException if there is an IO error reading the string
* @return UTF-8 string
*/
private static String readLongString(ByteArrayInputStream in) throws DataFormatException, IOException {
int size = in.read();
if (size < 0)
throw new EOFException("EOF reading string");
if (size == 0xff) {
size = (int) DataHelper.readLong(in, 2);
if (size > MAX_VALUE_LENGTH)
throw new DataFormatException(MAX_VALUE_LENGTH + " max, but this is " + size);
}
if (size == 0)
return "";
byte raw[] = new byte[size];
int read = DataHelper.read(in, raw);
if (read != size)
throw new EOFException("EOF reading string");
return new String(raw, "UTF-8");
}
/**
* Used to store entries that need deleting
*/
private static class InvalidEntry {
public final String key;
public final String list;
public InvalidEntry(String k, String l) {
key = k;
list = l;
}
}
/**
* BlockfileNamingService [force]
* force = force writable
*/
public static void main(String[] args) {
Properties ctxProps = new Properties();
if (args.length > 0 && args[0].equals("force"))
ctxProps.setProperty(PROP_FORCE, "true");
I2PAppContext ctx = new I2PAppContext(ctxProps);
BlockfileNamingService bns = new BlockfileNamingService(ctx);
Properties sprops = new Properties();
String lname = "privatehosts.txt";
sprops.setProperty("list", lname);
System.out.println("List " + lname + " contains " + bns.size(sprops));
lname = "userhosts.txt";
sprops.setProperty("list", lname);
System.out.println("List " + lname + " contains " + bns.size(sprops));
lname = "hosts.txt";
sprops.setProperty("list", lname);
System.out.println("List " + lname + " contains " + bns.size(sprops));
/****
List<String> names = null;
Properties props = new Properties();
try {
DataHelper.loadProps(props, new File("hosts.txt"), true);
names = new ArrayList(props.keySet());
Collections.shuffle(names);
} catch (IOException ioe) {
System.out.println("No hosts.txt to test with");
bns.close();
return;
}
System.out.println("size() reports " + bns.size());
System.out.println("getEntries() returns " + bns.getEntries().size());
System.out.println("Testing with " + names.size() + " hostnames");
int found = 0;
int notfound = 0;
int rfound = 0;
int rnotfound = 0;
long start = System.currentTimeMillis();
for (String name : names) {
Destination dest = bns.lookup(name);
if (dest != null) {
found++;
String reverse = bns.reverseLookup(dest);
if (reverse != null)
rfound++;
else
rnotfound++;
} else {
notfound++;
}
}
System.out.println("BFNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start));
System.out.println("found " + found + " notfound " + notfound);
System.out.println("reverse found " + rfound + " notfound " + rnotfound);
//if (true) return;
System.out.println("Removing all " + names.size() + " hostnames");
found = 0;
notfound = 0;
Collections.shuffle(names);
start = System.currentTimeMillis();
for (String name : names) {
if (bns.remove(name))
found++;
else
notfound++;
}
System.out.println("BFNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start));
System.out.println("removed " + found + " not removed " + notfound);
System.out.println("Adding back " + names.size() + " hostnames");
found = 0;
notfound = 0;
Collections.shuffle(names);
start = System.currentTimeMillis();
for (String name : names) {
try {
if (bns.put(name, new Destination(props.getProperty(name))))
found++;
else
notfound++;
} catch (DataFormatException dfe) {}
}
System.out.println("BFNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start));
System.out.println("Added " + found + " not added " + notfound);
System.out.println("size() reports " + bns.size());
//bns.dumpDB();
****/
bns.close();
ctx.logManager().flush();
System.out.flush();
/****
if (true) return;
HostsTxtNamingService htns = new HostsTxtNamingService(I2PAppContext.getGlobalContext());
found = 0;
notfound = 0;
start = System.currentTimeMillis();
for (String name : names) {
Destination dest = htns.lookup(name);
if (dest != null)
found++;
else
notfound++;
}
System.out.println("HTNS took " + DataHelper.formatDuration(System.currentTimeMillis() - start));
System.out.println("found " + found + " notfound " + notfound);
****/
}
}