/*
* Copyright 2011 Outerthought bvba
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.lilyproject.repository.impl;
import com.google.common.collect.Sets;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.Watcher.Event.EventType;
import org.apache.zookeeper.Watcher.Event.KeeperState;
import org.apache.zookeeper.data.Stat;
import org.lilyproject.repository.api.FieldType;
import org.lilyproject.repository.api.FieldTypes;
import org.lilyproject.repository.api.QName;
import org.lilyproject.repository.api.RecordType;
import org.lilyproject.repository.api.RepositoryException;
import org.lilyproject.repository.api.SchemaId;
import org.lilyproject.repository.api.TypeBucket;
import org.lilyproject.repository.api.TypeException;
import org.lilyproject.repository.api.TypeManager;
import org.lilyproject.util.Logs;
import org.lilyproject.util.Pair;
import org.lilyproject.util.zookeeper.ZkUtil;
import org.lilyproject.util.zookeeper.ZooKeeperItf;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.annotation.PreDestroy;
public abstract class AbstractSchemaCache implements SchemaCache {
protected ZooKeeperItf zooKeeper;
protected Log log = LogFactory.getLog(getClass());
private final CacheRefresher cacheRefresher = new CacheRefresher();
private FieldTypes fieldTypesSnapshot;
private FieldTypesCache fieldTypesCache = new FieldTypesCache();
private volatile boolean updatedFieldTypes = false;
private RecordTypesCache recordTypes = new RecordTypesCache();
private Set<CacheWatcher> cacheWatchers = Collections.synchronizedSet(new HashSet<CacheWatcher>());
private Map<String, Integer> bucketVersions = new ConcurrentHashMap<String, Integer>();
private ParentWatcher parentWatcher = new ParentWatcher();
private Integer parentVersion = null;
protected static final String CACHE_INVALIDATION_PATH = "/lily/typemanager/cache/invalidate";
protected static final String CACHE_REFRESHENABLED_PATH = "/lily/typemanager/cache/enabled";
private static final String LILY_NODES_PATH = "/lily/repositoryNodes";
/**
* Paths we need to watch for existence, since we need to re-initialize after they are recreated.
* Normally this doesn't happen, but it can happen with the resetLilyState() call of Lily's test
* framework.
*/
private static final Set<String> EXISTENCE_PATHS =
Sets.newHashSet(CACHE_INVALIDATION_PATH, CACHE_REFRESHENABLED_PATH, LILY_NODES_PATH);
public AbstractSchemaCache(ZooKeeperItf zooKeeper) {
this.zooKeeper = zooKeeper;
}
/**
* Used to build output as Hex
*/
private static final char[] DIGITS_LOWER = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd',
'e', 'f'};
private ConnectionWatcher connectionWatcher;
private LilyNodesWatcher lilyNodesWatcher;
/**
* Simplified version of {@link Hex#encodeHex(byte[])}
* <p/>
* In this version we avoid having to create a new byte[] to give to {@link Hex#encodeHex(byte[])}
*/
public static String encodeHex(byte[] data) {
char[] out = new char[2];
// two characters form the hex value.
out[0] = DIGITS_LOWER[(0xF0 & data[0]) >>> 4];
out[1] = DIGITS_LOWER[0x0F & data[0]];
return new String(out);
}
/**
* Decodes a string containing 2 characters representing a hex value.
* <p/>
* The returned byte[] contains the byte represented by the string and the next byte.
* <p/>
* This code is based on {@link Hex#decodeHex(char[])}
* <p/>
*/
public static byte[] decodeHexAndNextHex(String data) {
byte[] out = new byte[2];
// two characters form the hex value.
int f = Character.digit(data.charAt(0), 16) << 4;
f = f | Character.digit(data.charAt(1), 16);
out[0] = (byte) (f & 0xFF);
out[1] = (byte) ((f + 1) & 0xFF);
return out;
}
public static byte[] decodeNextHex(String data) {
byte[] out = new byte[1];
// two characters form the hex value.
int f = Character.digit(data.charAt(0), 16) << 4;
f = f | Character.digit(data.charAt(1), 16);
f++;
out[0] = (byte) (f & 0xFF);
return out;
}
public void start() throws InterruptedException, KeeperException, RepositoryException {
cacheRefresher.start();
ZkUtil.createPath(zooKeeper, CACHE_INVALIDATION_PATH);
final ExecutorService threadPool = Executors.newFixedThreadPool(50);
final List<Future> futures = new ArrayList<Future>();
for (int i = 0; i < 16; i++) {
final int index = i;
futures.add(threadPool.submit(new Callable<Void>() {
@Override
public Void call() throws InterruptedException, KeeperException, RepositoryException {
for (int j = 0; j < 16; j++) {
String bucket = "" + DIGITS_LOWER[index] + DIGITS_LOWER[j];
ZkUtil.createPath(zooKeeper, bucketPath(bucket));
cacheWatchers.add(new CacheWatcher(bucket));
}
return null;
}
}));
}
for (Future future : futures) {
try {
future.get();
} catch (ExecutionException e) {
throw new RuntimeException("failed to start cache", e);
}
}
threadPool.shutdown();
threadPool.awaitTermination(1, TimeUnit.HOURS);
ZkUtil.createPath(zooKeeper, CACHE_REFRESHENABLED_PATH);
connectionWatcher = new ConnectionWatcher();
zooKeeper.addDefaultWatcher(connectionWatcher);
readRefreshingEnabledState();
refreshAll();
}
private String bucketPath(String bucket) {
return CACHE_INVALIDATION_PATH + "/" + bucket;
}
@Override
@PreDestroy
public void close() throws IOException {
try {
zooKeeper.removeDefaultWatcher(connectionWatcher);
cacheRefresher.stop();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.debug("Interrupted", e);
}
}
/**
* Returns the typeManager which the cache can use to request for field
* types and record types from HBase instead of the cache.
*/
protected abstract TypeManager getTypeManager();
@Override
public FieldTypes getFieldTypesSnapshot() throws InterruptedException {
if (!updatedFieldTypes) {
return fieldTypesSnapshot;
}
synchronized (this) {
if (updatedFieldTypes) {
fieldTypesSnapshot = fieldTypesCache.getSnapshot();
updatedFieldTypes = false;
}
return fieldTypesSnapshot;
}
}
public void updateFieldType(FieldType fieldType) throws TypeException, InterruptedException {
fieldTypesCache.update(fieldType);
updatedFieldTypes = true;
}
public void updateRecordType(RecordType recordType) throws TypeException, InterruptedException {
recordTypes.update(recordType);
}
public Collection<RecordType> getRecordTypes() throws InterruptedException {
return recordTypes.getRecordTypes();
}
public RecordType getRecordType(QName name, Long version) throws InterruptedException {
return recordTypes.getRecordType(name, version);
}
public RecordType getRecordType(SchemaId id, Long version) {
return recordTypes.getRecordType(id, version);
}
public FieldType getFieldType(QName name) throws InterruptedException, TypeException {
return fieldTypesCache.getFieldType(name);
}
@Override
public Set<SchemaId> findDirectSubTypes(SchemaId recordTypeId) throws InterruptedException {
return recordTypes.findDirectSubTypes(recordTypeId);
}
public FieldType getFieldType(SchemaId id) throws TypeException, InterruptedException {
return fieldTypesCache.getFieldType(id);
}
public List<FieldType> getFieldTypes() throws TypeException, InterruptedException {
return fieldTypesCache.getFieldTypes();
}
public boolean fieldTypeExists(QName name) throws InterruptedException {
return fieldTypesCache.fieldTypeExists(name);
}
public FieldType getFieldTypeByNameReturnNull(QName name) throws InterruptedException {
return fieldTypesCache.getFieldTypeByNameReturnNull(name);
}
protected void readRefreshingEnabledState() {
// Should only do something on the HBaseTypeManager, not on the
// RemoteTypeManager
}
/**
* Refresh the caches and put the cacheWatcher again on the cache
* invalidation zookeeper-node.
*/
private void refreshAll() throws InterruptedException, RepositoryException {
watchPathsForExistence();
// Set a watch on the parent path, in case everything needs to be
// refreshed
try {
Stat stat = new Stat();
ZkUtil.getData(zooKeeper, CACHE_INVALIDATION_PATH, parentWatcher, stat);
if (parentVersion == null || (stat.getVersion() != parentVersion)) {
// An explicit refresh was triggered
parentVersion = stat.getVersion();
bucketVersions.clear();
}
} catch (KeeperException e) {
if (Thread.currentThread().isInterrupted()) {
if (log.isDebugEnabled()) {
log.debug("Failed to put parent watcher on " + CACHE_INVALIDATION_PATH + " : thread interrupted");
}
} else {
log.warn("Failed to put parent watcher on " + CACHE_INVALIDATION_PATH, e);
// Failed to put our watcher.
// Relying on the ConnectionWatcher to put it again and
// initialize the caches.
}
}
if (bucketVersions.isEmpty()) {
// All buckets need to be refreshed
if (log.isDebugEnabled()) {
log.debug("Refreshing all types in the schema cache, no bucket versions known yet");
}
// Set a watch again on all buckets
final ExecutorService sixteenThreads = Executors.newFixedThreadPool(50);
for (final CacheWatcher watcher : cacheWatchers) {
sixteenThreads.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
String bucketId = watcher.getBucket();
String bucketPath = bucketPath(bucketId);
Stat stat = new Stat();
try {
ZkUtil.getData(zooKeeper, bucketPath, watcher, stat);
bucketVersions.put(bucketId, stat.getVersion());
} catch (KeeperException e) {
if (Thread.currentThread().isInterrupted()) {
if (log.isDebugEnabled()) {
log.debug(
"Failed to put watcher on bucket " + bucketPath + " : thread interrupted");
}
} else {
log.warn("Failed to put watcher on bucket " + bucketPath
+ " - Relying on connection watcher to reinitialize cache", e);
// Failed to put our watcher.
// Relying on the ConnectionWatcher to put it again and
// initialize the caches.
}
}
return null;
}
});
}
sixteenThreads.shutdown();
sixteenThreads.awaitTermination(1, TimeUnit.HOURS);
// Read all types in one go
Pair<List<FieldType>, List<RecordType>> types = getTypeManager().getTypesWithoutCache();
fieldTypesCache.refreshFieldTypes(types.getV1());
updatedFieldTypes = true;
recordTypes.refreshRecordTypes(types.getV2());
} else {
// Only the changed buckets need to be refreshed.
// Upon a re-connection event it could be that some updates were
// missed and the watches were not triggered.
// By checking the version number of the buckets we know which
// buckets to refresh.
Map<String, Integer> newBucketVersions = new HashMap<String, Integer>();
// Set a watch again on all buckets
for (CacheWatcher watcher : cacheWatchers) {
String bucketId = watcher.getBucket();
String bucketPath = bucketPath(bucketId);
Stat stat = new Stat();
try {
ZkUtil.getData(zooKeeper, bucketPath, watcher, stat);
Integer oldVersion = bucketVersions.get(bucketId);
if (oldVersion == null || (oldVersion != stat.getVersion())) {
newBucketVersions.put(bucketId, stat.getVersion());
}
} catch (KeeperException e) {
if (Thread.currentThread().isInterrupted()) {
if (log.isDebugEnabled()) {
log.debug("Failed to put watcher on bucket " + bucketPath + " : thread is interrupted");
}
} else {
log.warn("Failed to put watcher on bucket " + bucketPath
+ " - Relying on connection watcher to reinitialize cache", e);
// Failed to put our watcher.
// Relying on the ConnectionWatcher to put it again and
// initialize the caches.
}
}
}
if (log.isDebugEnabled()) {
log.debug("Refreshing all types in the schema cache, limiting to buckets" + newBucketVersions.keySet());
}
for (Entry<String, Integer> entry : newBucketVersions.entrySet()) {
bucketVersions.put(entry.getKey(), entry.getValue());
TypeBucket typeBucket = getTypeManager().getTypeBucketWithoutCache(entry.getKey());
fieldTypesCache.refreshFieldTypeBucket(typeBucket);
updatedFieldTypes = true;
recordTypes.refreshRecordTypeBucket(typeBucket);
}
}
}
/**
* Refresh the caches for the buckets identified by the watchers and put the
* cacheWatcher again on the cache invalidation zookeeper-node.
*/
private void refresh(Set<CacheWatcher> watchers) throws InterruptedException, RepositoryException {
// Only update one bucket at a time
// Meanwhile updates on the other buckets could happen.
// Since the watchers for those other buckets are not set back again
// this will not trigger extra refreshes.
boolean first = true;
for (CacheWatcher watcher : watchers) {
// Throttle the refreshing. When a burst of updates occur, delaying
// the refreshing a bit allows for the updates to be performed
// faster
if (first) {
first = false;
Thread.sleep(50);
}
String bucketId = watcher.getBucket();
String bucketPath = bucketPath(watcher.getBucket());
Stat stat = new Stat();
try {
ZkUtil.getData(zooKeeper, bucketPath, watcher, stat);
if (stat.getVersion() == bucketVersions.get(bucketId)) {
continue; // The bucket is up to date
}
} catch (KeeperException e) {
if (Thread.currentThread().isInterrupted()) {
if (log.isDebugEnabled()) {
log.debug("Failed to put watcher on bucket " + bucketPath + " : thread is interrupted");
}
} else {
log.warn("Failed to put watcher on bucket " + bucketPath
+ " - Relying on connection watcher to reinitialize cache", e);
// Failed to put our watcher.
// Relying on the ConnectionWatcher to put it again and
// initialize the caches.
}
}
if (log.isDebugEnabled()) {
log.debug("Refreshing schema cache bucket: " + bucketId);
}
// Avoid updating the cache while refreshing the buckets
bucketVersions.put(bucketId, stat.getVersion());
TypeBucket typeBucket = getTypeManager().getTypeBucketWithoutCache(bucketId);
fieldTypesCache.refreshFieldTypeBucket(typeBucket);
recordTypes.refreshRecordTypeBucket(typeBucket);
}
updatedFieldTypes = true;
}
private void watchPathsForExistence() throws InterruptedException {
for (String path : EXISTENCE_PATHS) {
try {
zooKeeper.exists(path, true);
} catch (KeeperException e) {
if (Thread.currentThread().isInterrupted()) {
if (log.isDebugEnabled()) {
log.debug("Failed to put existence watcher on " + path + " : thread is interrupted");
}
} else {
log.warn("Failed to put existence watcher on " + path, e);
}
}
}
}
/**
* Cache refresher refreshes the cache when flagged.
* <p/>
* The {@link CacheWatcher} monitors the cache invalidation flag on Zookeeper. When this flag changes it will call
* {@link #needsRefresh} on the CacheRefresher, setting the needsRefresh flag. This is the only thing the
* CacheWatcher does. Thereby it can return quickly when it received an event.<br/>
* <p/>
* The CacheRefresher in its turn will notice the needsRefresh flag being set and will refresh the cache. It runs
* in
* a separate thread so that we can avoid that the refresh work would be done in the thread of the watcher.
*/
private class CacheRefresher implements Runnable {
private volatile boolean needsRefresh;
private volatile boolean needsRefreshAll;
private volatile boolean lilyNodesChanged;
// we do not rely on thread interruption alone because some libraries "eat" interrupted exceptions
private volatile boolean running;
private final Object needsRefreshLock = new Object();
private Set<CacheWatcher> needsRefreshWatchers = new HashSet<CacheWatcher>();
private Thread thread;
private List<String> knownLilyNodes = new ArrayList<String>();
public void start() {
if (running) {
if (log.isDebugEnabled()) {
log.debug("not starting because already running");
}
} else {
running = true;
thread = new Thread(this, "TypeManager cache refresher");
thread.setDaemon(true); // Since this might be used in clients
thread.start();
}
}
public void stop() throws InterruptedException {
if (running) {
running = false;
if (thread != null) {
thread.interrupt();
Logs.logThreadJoin(thread);
thread.join();
thread = null;
}
}
}
public void needsRefreshAll() {
synchronized (needsRefreshLock) {
needsRefreshAll = true;
needsRefreshLock.notifyAll();
}
}
public void needsRefresh(CacheWatcher watcher) {
synchronized (needsRefreshLock) {
needsRefresh = true;
needsRefreshWatchers.add(watcher);
needsRefreshLock.notifyAll();
}
}
public void lilyNodesChanged() {
synchronized (needsRefreshLock) {
lilyNodesChanged = true;
needsRefreshLock.notifyAll();
}
}
private List<String> getLilyNodes() throws KeeperException, InterruptedException {
LilyNodesWatcher watcher = lilyNodesWatcher;
if (watcher == null) {
watcher = new LilyNodesWatcher();
}
List<String> lilyNodes = null;
try {
lilyNodes = zooKeeper.getChildren(LILY_NODES_PATH, watcher);
lilyNodesWatcher = watcher;
} catch (KeeperException e) {
if (log.isDebugEnabled()) {
log.debug("Failed getting lilyNodes from Zookeeper", e);
}
// The path does not exist yet.
// Set the lilyNodesWatcher to null so that we retry
// setting the watcher in the next iteration.
lilyNodesWatcher = null;
}
return lilyNodes;
}
@Override
public void run() {
while (running && !Thread.interrupted()) {
try {
// Check if the lily nodes changed
// or if the LilyNodesWatcher has not been set yet
if (lilyNodesChanged || lilyNodesWatcher == null) {
synchronized (needsRefreshLock) {
List<String> lilyNodes = getLilyNodes();
if (lilyNodes != null) {
if (knownLilyNodes.isEmpty()) {
knownLilyNodes.addAll(lilyNodes);
} else {
if (!lilyNodes.containsAll(knownLilyNodes)) {
// One or more of the nodes disappeared.
// There is a chance that a refresh
// trigger was not sent out by that
// node.
needsRefreshAll = true;
// Set parentVersion to null to
// explicitly refresh all buckets.
// Otherwise only buckets that got an
// update trigger will be refreshed.
// But because such a trigger could have
// been missed is why we are here.
parentVersion = null;
if (log.isDebugEnabled()) {
log.debug("One or more LilyNodes stopped. "
+
"Refreshing cache to cover possibly missed refresh triggers");
}
}
knownLilyNodes.clear();
knownLilyNodes.addAll(lilyNodes);
}
} else if (!knownLilyNodes.isEmpty()) {
needsRefreshAll = true;
knownLilyNodes.clear();
}
lilyNodesChanged = false;
}
}
// Check if something needs to be refreshed
if (needsRefresh || needsRefreshAll) {
Set<CacheWatcher> watchers = null;
boolean refreshAll = false;
synchronized (needsRefreshLock) {
if (needsRefreshAll) {
refreshAll = true;
} else {
watchers = new HashSet<CacheWatcher>(needsRefreshWatchers);
}
needsRefreshWatchers.clear();
needsRefresh = false;
needsRefreshAll = false;
}
// Do the actual refresh outside the synchronized block
if (refreshAll) {
refreshAll();
} else {
refresh(watchers);
}
}
synchronized (needsRefreshLock) {
if (!needsRefresh && !needsRefreshAll && running) {
needsRefreshLock.wait();
}
}
} catch (InterruptedException e) {
return;
} catch (Throwable t) {
// Sometimes InterruptedException is wrapped in IOException (from hbase)
// or RemoteException (from avro), and sometimes these by themselves are again
// wrapped in a TypeException of Lily.
Throwable cause = t.getCause();
while (cause != null) {
if (cause instanceof InterruptedException) {
return;
}
cause = cause.getCause();
}
log.error("Error refreshing type manager cache. Cache is possibly out of date!", t);
}
}
}
}
/**
* The Cache watcher monitors events on the cache invalidation flag.
*/
private class CacheWatcher implements Watcher {
private final String bucket;
CacheWatcher(String bucket) {
this.bucket = bucket;
}
public String getBucket() {
return bucket;
}
@Override
public void process(WatchedEvent event) {
if (EventType.NodeDataChanged.equals(event.getType())) {
cacheRefresher.needsRefresh(this);
}
}
}
/**
* This watcher will be triggered when an explicit refresh is requested.
* <p/>
* It is put on the parent path: CACHE_INVALIDATION_PATH
*
* @see {@link TypeManager#triggerSchemaCacheRefresh()}
*/
private class ParentWatcher implements Watcher {
@Override
public void process(WatchedEvent event) {
if (EventType.NodeDataChanged.equals(event.getType())) {
cacheRefresher.needsRefreshAll();
}
}
}
/**
* The ConnectionWatcher monitors for zookeeper (re-)connection events.
* <p/>
* It will set the cache invalidation flag if needed and refresh the caches since events could have been missed
* while being disconnected.
*/
protected class ConnectionWatcher implements Watcher {
@Override
public void process(WatchedEvent event) {
if (EventType.None.equals(event.getType()) && KeeperState.SyncConnected.equals(event.getState())) {
readRefreshingEnabledState();
cacheRefresher.needsRefreshAll();
} else if (EXISTENCE_PATHS.contains(event.getPath())) {
if (EventType.NodeCreated.equals(event.getType())) {
if (event.getPath().equals(CACHE_REFRESHENABLED_PATH)) {
readRefreshingEnabledState();
}
if (event.getPath().equals(CACHE_INVALIDATION_PATH)) {
// Handle things to survive resetLilyState:
// - ZK bucket version numbers won't be relevant anymore
bucketVersions.clear();
// - Since the cache invalidation path is new, the schema situation is new,
// forget what is currently in the caches
fieldTypesCache.clear();
recordTypes.clear();
cacheRefresher.needsRefreshAll();
}
if (event.getPath().equals(LILY_NODES_PATH)) {
lilyNodesWatcher = null;
cacheRefresher.needsRefreshAll();
}
} else if (EventType.NodeDeleted.equals(event.getType())) {
try {
watchPathsForExistence();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
}
}
}
}
}
protected class LilyNodesWatcher implements Watcher {
@Override
public void process(WatchedEvent event) {
cacheRefresher.lilyNodesChanged();
}
}
}