/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/* This file contains code under the Apache 2 license - see hashFNV */
package org.apache.jena.tdb.index.ext;
import static java.lang.String.format;
import java.nio.IntBuffer;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import org.apache.jena.atlas.io.IndentedLineBuffer ;
import org.apache.jena.atlas.io.IndentedWriter ;
import org.apache.jena.atlas.lib.BitsLong ;
import org.apache.jena.atlas.lib.Bytes ;
import org.apache.jena.atlas.lib.Lib ;
import org.apache.jena.tdb.base.StorageException ;
import org.apache.jena.tdb.base.block.BlockMgr ;
import org.apache.jena.tdb.base.block.BlockMgrFactory ;
import org.apache.jena.tdb.base.buffer.RecordBuffer ;
import org.apache.jena.tdb.base.file.PlainFile ;
import org.apache.jena.tdb.base.file.PlainFileMem ;
import org.apache.jena.tdb.base.record.Record ;
import org.apache.jena.tdb.base.record.RecordFactory ;
import org.apache.jena.tdb.index.Index ;
import org.apache.jena.tdb.sys.SystemTDB ;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** Extensible hashing
* http://en.wikipedia.org/wiki/Extendible_hashing
*/
public final class ExtHash implements Index
{
/* Hashing.
* Extendible hashing is based on taking more of the bits of the hash
* value to address an expanding dictionary. This is a bit-trie, stored
* as an array. One bucket can be used for several hash slots.
*
* We need that the bits are used in decreasing signifance because we
* keep items in buckets in full-hash sorted order.
*
* Side effect: the whole structure is sorted by full hash, using
* dictionary and buckets.
*
* But.
* Java .hashCode() does not make suitable hash directly because either
* they are Object.hashCode (not too bad but it tends not to use high bits) or
* something like Integer.hashCode is the integer value itself. The
* latter is very bad as the hash is not using the high bits (most
* integers are small - especially sequentially allocated numbers).
*
* Solution: use the hashCode, 31 bits (arrays indexes are signed)
* but bit reversed so low bits of the original value are the most
* significant (when shorter than the currect hash, it's the low bits
* that are used).
*
*
*
* Example: using hex chars, not bits.
*
* Record: key: 0xABCD
* Length = 1 ==> trie is D
* if the length changes from 1 to 2,
* Length = 2 ==> trie is DC, that is, the D is most significant.
* and all buckets Dx point to what was in slot for D.
*
* All hash handling is encapulated in the internal routines.
*/
static private Logger log = LoggerFactory.getLogger(ExtHash.class) ;
// Production: make these final and false.
public static boolean Debugging = false ;
public static boolean Checking = false ; // Perform internal checking
public static boolean Logging = false ; // Allow any logging code on critical paths
IntBuffer dictionary ; // mask(hash) -> Bucket id
// Current length of trie bit used. Invariant: dictionary.length = 1<<bitLen
private int bitLen = 0 ;
private final HashBucketMgr hashBucketMgr ;
private final RecordFactory recordFactory ;
private final PlainFile dictionaryFile ;
/** Testing version - in-memory but inefficient as it uses a copy-in/copy-out block manager as a RAM disk*/
static public ExtHash createMem(RecordFactory factory, int bucketSizeBytes )
{
BlockMgr mgr = BlockMgrFactory.createMem("ExtHash", bucketSizeBytes) ;
ExtHash eHash = new ExtHash(new PlainFileMem(), factory, mgr) ;
return eHash ;
}
public ExtHash(PlainFile dictionaryBackingFile,
RecordFactory recordFactory, BlockMgr blockMgrHashBuckets)
{
this.dictionaryFile = dictionaryBackingFile ;
// Start bigger?
int dictionarySize = 1 ;
dictionary = dictionaryFile.ensure(SystemTDB.SizeOfInt).asIntBuffer() ;
this.recordFactory = recordFactory ;
hashBucketMgr = new HashBucketMgr(recordFactory, blockMgrHashBuckets) ;
hashBucketMgr.valid(0) ;
// Did it exist?
if ( hashBucketMgr.valid(0) )
{
}
else
{
// int id = hashBucketMgr.allocateId() ;
// if ( id != 0 )
// throw new StorageException("ExtHash: First bucket is not id zero") ;
HashBucket hb = hashBucketMgr.create(0, 0) ;
dictionary.put(0, hb.getId()) ;
bitLen = 0 ;
hashBucketMgr.put(hb) ;
}
}
// =====================
// Hashing routines for converting to a bit-trie (i.e. lowest bit
// is most significant in the trie).
// It's important that bits "appear" as the length changes in decreasing
// significance order - we reveal bits by lengthing the key up the integer.
// Use 31 bits (not the top hashCode bit) because array indexes are signed 32bit.
// See java.util.HashMap#hash for discussion of supplemental hashing.
interface HashRecordKey { public int hashCode(byte[] key) ; }
private HashRecordKey hashFunction = hashFNV ; //hash4bytes ;
// Hash function that is the first 4 bytes of the key (key must be at least 4 bytes long).
static HashRecordKey hash4bytes = new HashRecordKey(){
@Override
public final int hashCode(byte[] key)
{ return Bytes.getInt(key) ; }
} ;
/** From Project Voldemort / Apache License / Thanks!
* who in turn got it from
*
* Taken from http://www.isthe.com/chongo/tech/comp/fnv
*
* hash = basis for each octet_of_data to be hashed hash
* = hash * FNV_prime hash
* = hash xor octet_of_data return hash
*
*/
static HashRecordKey hashFNV = new HashRecordKey(){
private static final long FNV_BASIS = 0x811c9dc5;
private static final long FNV_PRIME = (1 << 24) + 0x193;
@Override
public final int hashCode(byte[] key)
{
long hash = FNV_BASIS;
for ( byte aKey : key )
{
hash ^= 0xFF & aKey;
hash *= FNV_PRIME;
}
return (int) hash;
}
} ;
/** Turn a key into a bit trie hash value */
private int trieKey(Record k)
{
// ***** Record key to hash: high bits most significant
// Does not have to be a perfect hash.
int x = hashFunction.hashCode(k.getKey()) ;
return Integer.reverse(x)>>>1 ;
}
/** Calculate the array index for a key given the dictionary bit length */
private int trieKey(Record key, int bitLen) { return trieKey(trieKey(key), bitLen) ; }
/** Convert from full hash to array index for a dictionary bit length */
private int trieKey(int fullTrie, int bitLen) { return fullTrie >>> (31-bitLen) ; }
/** Calculate the bucket id for a key given the dictionary bit length */
private int bucketId(Record key, int bitLen)
{
int x = trieKey(trieKey(key), bitLen) ;
int id = dictionary.get(x) ;
return id ;
}
/** Size of the file, in bytes */
private static long filesize(int dictionarySize) { return 4L*dictionarySize ; }
// =====================
private void resizeDictionary()
{
int oldSize = 1<<bitLen ;
int newBitLen = bitLen+1 ;
int newSize = 1<<newBitLen ;
if ( logging() )
{
log(">>>>Resize") ;
log("resize: %d ==> %d", oldSize, newSize) ;
}
IntBuffer newDictionary = dictionaryFile.ensure(newSize*SystemTDB.SizeOfInt).asIntBuffer() ;
if ( dictionary != null )
{
// Fill new dictionary
// NB Fills from high to low so that it works "in place"
for ( int i = oldSize-1 ; i>=0 ; i-- )
{
int b = newDictionary.get(i) ;
//if ( logging() ) log("Resize: put: (%d, %d)", 2*i, b) ;
newDictionary.put(2*i, b) ;
newDictionary.put(2*i+1, b) ;
}
}
dictionary = newDictionary ;
bitLen = newBitLen ;
if ( logging() )
{
if ( false ) dump() ;
if ( false ) log(this) ;
log("<<<<Resize") ;
}
internalCheck() ;
}
// =====================
// Used by the iterator
final int getBucketId(int dictionaryIdx)
{
return dictionary.get(dictionaryIdx) ;
}
final HashBucket getBucket(int blockId)
{
return hashBucketMgr.get(blockId) ;
}
public final int dictionarySize()
{
return dictionary.capacity() ;
}
// =====================
@Override
public boolean contains(Record key)
{
return find(key) != null ;
}
@Override
public Record find(Record key)
{
if ( logging() ) log(">> get(%s)", key) ;
int blockId = bucketId(key, bitLen) ;
HashBucket bucket = hashBucketMgr.get(blockId) ;
Record value = bucket.find(key) ;
if ( logging() ) log("<< get(%s) -> %s", key.getKey(), value) ;
return value ;
}
@Override
public boolean add(Record record)
{
if ( logging() ) log(">> add(%s)", record) ;
int h = trieKey(record) ;
boolean b = put(record, h) ;
if ( logging() )
{
log("<< add(%s)", record) ;
//dump() ;
}
internalCheck() ;
return b ;
}
@Override
public boolean delete(Record record)
{
if ( logging() ) log(">> remove(%s)", record) ;
int blockId = bucketId(record, bitLen) ;
HashBucket bucket = hashBucketMgr.get(blockId) ;
boolean b = bucket.removeByKey(record) ;
hashBucketMgr.put(bucket) ;
internalCheck() ;
if ( logging() ) log("<< remove(%s)", record) ;
return b ;
}
@Override
public RecordFactory getRecordFactory()
{ return recordFactory ; }
@Override
public Iterator<Record> iterator()
{
return new ExtHashIterator(this) ;
}
@Override
public boolean isEmpty()
{
if ( dictionary.limit() == 1 )
{
HashBucket b = hashBucketMgr.get(1) ;
return b.isEmpty() ;
}
// No idea.
return false ;
}
@Override
public void clear()
{ throw new UnsupportedOperationException("RangeIndex("+Lib.classShortName(this.getClass())+").clear") ; }
@Override
public long size()
{ return count() ; }
/** Explicitly count the items in the hash table */
public long count()
{
Set<Integer> seen = new HashSet<>() ;
long count = 0 ;
for ( int i = 0 ; i < dictionary.capacity() ; i++ )
{
int id = dictionary.get(i) ;
if ( seen.contains(id) )
continue ;
seen.add(id) ;
HashBucket bucket = hashBucketMgr.get(id) ;
count += bucket.getCount() ;
}
return count ;
}
@Override
public void sync()
{
hashBucketMgr.getBlockMgr().sync() ;
dictionaryFile.sync() ;
}
@Override
public void close()
{
hashBucketMgr.getBlockMgr().close() ;
dictionaryFile.close() ;
}
// =====================
// Insert
// Reentrant part of "put"
private boolean put(Record record, int hash)
{
if ( logging() ) log("put(%s,0x%08X)", record, hash) ;
int dictIdx = trieKey(hash, bitLen) ; // Dictionary index
int blockId = dictionary.get(dictIdx) ;
HashBucket bucket = hashBucketMgr.get(blockId) ;
if ( ! bucket.isFull() )
{
if ( Debugging ) log("Insert [(0x%04X) %s]: %d", hash, record, bucket.getId()) ;
boolean b = bucket.put(record) ;
hashBucketMgr.put(bucket) ;
return b ;
}
//Is this and +1 the same? Is the block splitable?
// Bucket full.
if ( bitLen == bucket.getTrieBitLen() )
{
// // Log it anyway
// if ( ! logging() ) log("put(%s,0x%08X)", record, hash) ;
boolean oldLogging = Logging ;
boolean oldDebugging = Debugging ;
try {
// Logging = true ;
// Debugging = true ;
if ( Debugging )
{
log("Bucket full: %d", bucket.getId()) ;
log("Bucket can't be split - dictionary resize needed") ;
//log(bucket) ;
this.dump() ;
}
// Bucket not splitable..
// TODO Overflow buckets.
// Expand the dictionary.
int x = dictionarySize() ;
resizeDictionary() ;
if ( Debugging ) log("Resize: %d -> %d", x, dictionarySize()) ;
// Try again
return put(record, hash) ;
} finally { Logging = oldLogging ; Debugging = oldDebugging ;}
}
if ( Debugging ) log("Split bucket: %d", bucket.getId()) ;
// bitLen > bucket.getHashBitLen() : bucket can be split
splitAndReorganise(bucket, dictIdx, blockId, hash) ;
// Reorg done - try again.
return put(record, hash) ;
}
// Bucket bitlength is less than that of the dictionary.
private void splitAndReorganise(HashBucket bucket, int dictionaryIdx, int bucketId, int hash)
{
if ( logging() )
{
log("splitAndReorganise: idx=%d, id=%d, bitLen=%d, bucket.hashLength=%d",
dictionaryIdx, bucketId, bitLen, bucket.getTrieBitLen()) ;
if ( false ) dump() ;
}
if ( Checking )
{
if ( bucket.getTrieBitLen() >= bitLen )
error("splitAndReorganise: idx=0x%X : hash=0x%X[0x%X,0x%X] : Hash not shorter : %s",
dictionaryIdx, hash, trieKey(hash, bucket.getTrieBitLen()), bucket.getTrieValue(), bucket) ;
if ( trieKey(hash, bucket.getTrieBitLen()) != bucket.getTrieValue() )
error("splitAndReorganise: idx=0x%X : hash=0x%X[0x%X,0x%X] : Inconsistency : %s",
dictionaryIdx, hash, trieKey(hash, bucket.getTrieBitLen()), bucket.getTrieValue(), bucket) ;
}
// Bucket did not have a full length hash so split it.
// Find the companion slots.
// Remember before messing with split.
int bucketHash = bucket.getTrieValue() ;
int bucketHashLength = bucket.getTrieBitLen() ;
// Split the bucket in two. bucket2 is the upper bucket.
HashBucket bucket2 = split(bucketId, bucket) ;
// Determine the slots affected:
// All the dictionary entries that in the extension of the bit trie, have a 1
// in the newly exposed bit. These will point to bucket2.
// All the slots for bit 0 will continue to point to the existing (reorganised) bucket.
// The hash is reversed (the low bits the hash value are most significant).
// So extending a hash is shift up, and OR in a 0 or 1.
// Zeros in the difference the bucket bit length and the dictionary bitlength.
// Upper section of bucket hash, extended, then the gap in lengths zero filled.
int trieUpperRoot = ((bucketHash<<1)|0x1) << (bitLen-bucketHashLength-1) ;
// Upper bound (exclusive) of values affected between dictionary and current bucket.
// NB relationship to the second shift on trieUpperRoot
int trieUpperRange = (1<<(bitLen-bucketHashLength-1)) ;
for ( int j = 0 ; j < trieUpperRange ; j++ )
{
// j runs over the values of the unused bits of the trie start for the upper bucket positions.
int k = trieUpperRoot | j ;
if ( logging() )
log("Point to split bucket: 0x%04X", k) ;
if ( Checking )
{
if ( (trieUpperRoot&j) != 0 )
error("put: idx=%d : trieRoot=0x%X, sub=%d: Broken trie pattern ", dictionaryIdx, trieUpperRoot, j) ;
if ( ! BitsLong.isSet(k, (bitLen-(bucketHashLength+1)) ) )
error("put: Broken trie pattern (0x%X,%d)", trieUpperRoot, j) ;
// We should looking at the original bucket
int id = dictionary.get(k) ;
HashBucket hb = hashBucketMgr.get(id) ;
if ( hb.getId() != bucket.getId() )
error("put: Wrong bucket at trie 0x%X %d: (%d,%d)", trieUpperRoot, j, hb.getId(), bucket.getId()) ;
}
dictionary.put(k, bucket2.getId()) ;
}
if ( logging() )
{
log("Reorg complete") ;
if ( false ) dump() ;
}
}
private HashBucket split(int bucketId, HashBucket bucket)
{
// idx is the array offset to the lower of the bucket point pair.
if ( logging() )
{
log("split: Bucket %d : size: %d; Bucket bitlength %d", bucketId, bucket.getCount(), bucket.getTrieBitLen()) ;
log("split: %s", bucket) ;
}
// Create new bucket, which will be the upper bucket.
// Low bucket will have the old hash value,
// Lengthen the hash; the new will be one more.
bucket.incTrieBitLen() ;
// Bucket hash value is kept in index-order (i.e. high bits are most significant).
int hash1 = bucket.getTrieValue() << 1 ;
int hash2 = (bucket.getTrieValue() << 1) | 0x1 ;
// Reset, now it's longer
bucket.setTrieValue(hash1) ;
if ( logging() )
log("split: bucket hashes 0x%04X 0x%04X", hash1, hash2) ;
// // New bucket
HashBucket bucket2 = hashBucketMgr.create(hash2, bucket.getTrieBitLen()) ;
if ( logging() ) log("New bucket: %s", bucket2) ;
//bucket2.setTrieValue(hash2) ;
RecordBuffer rBuff1 = bucket.getRecordBuffer() ;
RecordBuffer rBuff2 = bucket2.getRecordBuffer() ;
int idx1 = 0 ; // Destination indexes into the above
int idx2 = 0 ;
for ( int i = 0 ; i < rBuff1.size() ; i++ )
{
Record r = rBuff1.get(i) ;
int x = trieKey(r, bucket.getTrieBitLen()) ; // Incremented bit length
if ( x == hash1 )
{
if ( logging() )
log("Allocate index %d to bucket1", i) ;
// idx1 <= i (we are writing less records back).
// So this foes not interfer with the loop
// We're shifting down records that saty in this bucket.
if ( idx1 != i )
rBuff1.set(idx1, r) ;
idx1++ ;
}
else if ( x == hash2 )
{
if ( logging() )
log("Allocate index %d to bucket2", i) ;
rBuff2.add(r) ;
idx2 ++ ;
}
else
error("Bad trie for allocation to split buckets") ;
}
if ( true )
rBuff1.clear(idx1, bucket.getCount()-idx1) ;
rBuff1.setSize(idx1) ;
// rBuff2 was fresh so still clean.
if ( logging() )
{
log("split: Lower bucket: %s", bucket) ;
log("split: Upper bucket: %s", bucket2) ;
}
// Check with splitAndReorganise()
hashBucketMgr.put(bucket) ;
hashBucketMgr.put(bucket2) ;
return bucket2 ;
}
// =====================
@Override
public String toString()
{
IndentedLineBuffer buff = new IndentedLineBuffer() ;
dump(buff) ;
return buff.asString() ;
}
public void dump()
{
dump(IndentedWriter.stdout) ;
IndentedWriter.stdout.ensureStartOfLine() ;
IndentedWriter.stdout.flush() ;
}
private void dump(IndentedWriter out)
{
out.printf("Bitlen = %d\n" , bitLen) ;
out.printf("Dictionary = %d\n" , 1<<bitLen ) ;
out.incIndent(4) ;
for ( int i = 0 ; i < (1<<bitLen) ; i++ )
{
out.ensureStartOfLine() ;
int id = dictionary.get(i) ;
HashBucket bucket = hashBucketMgr.get(id) ;
out.printf("[%d] %02d %s", i, id, bucket) ;
}
out.decIndent(4) ;
}
@Override
public void check()
{
performCheck() ;
}
private final void internalCheck()
{
if ( Checking )
performCheck() ;
}
private final void performCheck()
{
int len = 1<<bitLen ;
int d = dictionary.limit() ;
//int d = (dictionary.limit()/4) ;
if ( len != d )
error("Dictionary size = %d : expected = %d", d, len) ;
Set<Integer> seen = new HashSet<>() ;
for ( int i = 0 ; i < d ; i++ )
{
int id = dictionary.get(i) ;
if ( seen.contains(id) )
continue ;
seen.add(id) ;
HashBucket bucket = hashBucketMgr.get(id) ;
performCheck(i, bucket) ;
}
}
private void performCheck(int idx, HashBucket bucket)
{
if ( bucket.getTrieBitLen() > bitLen )
error("[%d] Bucket %d has bit length longer than the dictionary's (%d, %d)", idx, bucket.getId(), bucket.getTrieBitLen(), bitLen) ;
// Check the bucket hash against the slot it's in.
// Convert directory index to bucket hash
int tmp = (idx >>> (bitLen-bucket.getTrieBitLen())) ;
if ( tmp != bucket.getTrieValue())
error("[%d] Bucket %d : hash prefix 0x%X, expected 0x%X : %s", idx, bucket.getId(), bucket.getTrieValue(), tmp, bucket) ;
// Check the contents.
Record prevKey = Record.NO_REC ;
for ( int i = 0 ; i < bucket.getCount() ; i++ )
{
Record rec = bucket.get(i) ;
if ( prevKey != Record.NO_REC && Record.keyLT(rec,prevKey) )
error("[%d] Bucket %d: Not sorted (slot %d) : %s", idx, bucket.getId(), i, bucket) ;
prevKey = rec ;
int x = trieKey(rec, bucket.getTrieBitLen()) ;
// Check the key is bucket-compatible.
if ( x != bucket.getTrieValue() )
error("[%d] Bucket %d: Key (0x%04X) does not match the hash (0x%04X) : %s",
idx, bucket.getId(), x, bucket.getTrieValue(), bucket) ;
}
if ( SystemTDB.NullOut )
{
for ( int i = bucket.getCount() ; i < bucket.getMaxSize() ; i++ )
{
if ( ! bucket.getRecordBuffer().isClear(i) )
error("[%d] Bucket %d : overspill at [%d]: %s", idx, bucket.getId(), i, bucket) ;
}
}
}
private void error(String msg, Object... args)
{
msg = format(msg, args) ;
log.error(msg) ;
throw new StorageException(msg) ;
}
private final boolean logging() { return Logging /* && log.isDebugEnabled()*/ ; }
private final void log(String format, Object... args)
{
//if ( ! logging() ) return ;
log.debug(format(format, args)) ;
}
private final void log(Object obj)
{
//if ( ! logging() ) return ;
log.debug(obj.toString()) ;
}
}