/*
* Copyright 2013 Gordon Burgett and individual contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.xflatdb.xflat.db;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jdom2.Document;
import org.jdom2.Element;
import org.xflatdb.xflat.Database;
import org.xflatdb.xflat.DatabaseBuilder;
import org.xflatdb.xflat.DatabaseConfig;
import org.xflatdb.xflat.KeyValueTable;
import org.xflatdb.xflat.Table;
import org.xflatdb.xflat.TableConfig;
import org.xflatdb.xflat.XFlatConstants;
import org.xflatdb.xflat.XFlatException;
import org.xflatdb.xflat.convert.ConversionException;
import org.xflatdb.xflat.convert.ConversionService;
import org.xflatdb.xflat.convert.DefaultConversionService;
import org.xflatdb.xflat.convert.PojoConverter;
import org.xflatdb.xflat.convert.converters.DateConverters;
import org.xflatdb.xflat.convert.converters.JDOMConverters;
import org.xflatdb.xflat.convert.converters.StringConverters;
import org.xflatdb.xflat.engine.DefaultEngineFactory;
import org.xflatdb.xflat.transaction.ThreadContextTransactionManager;
import org.xflatdb.xflat.transaction.TransactionManager;
import org.xflatdb.xflat.util.Action1;
import org.xflatdb.xflat.util.DocumentFileWrapper;
/**
* This database implementation manages a local directory of tables. It is the
* main database implementation inside XFlat.
* <p/>
* Within the directory, each table is represented by an XML file. If the table
* is sharded, it is represented as a subdirectory, in which each shard is represented
* as an XML file.
* <p/>
* This implementation supports the following requirements:<br/>
* <pre>
* "transactional" : true | [ A, C, I, D ]
* "local" : true
* "threadsafe" : true
* </pre>
* @author gordon
*/
public class XFlatDatabase implements Database {
//<editor-fold desc="dependencies">
private ScheduledExecutorService executorService;
protected ScheduledExecutorService getExecutorService(){
return executorService;
}
private ConversionService conversionService;
/**
* Gets the current conversion service. The conversion service can be
* set or updated in order to convert any objects.
* @return the database's conversion service.
*/
public ConversionService getConversionService() {
return this.conversionService;
}
private EngineFactory engineFactory = new DefaultEngineFactory();
/**
* Sets the {@link EngineFactory} used to create {@link Engine} instances
* for the tables.
* @param factory
*/
public void setEngineFactory(EngineFactory factory){
this.engineFactory = factory;
}
/**
* @see #setEngineFactory(org.xflatdb.xflat.db.EngineFactory)
*/
public EngineFactory getEngineFactory(){
return this.engineFactory;
}
private TableMetadataFactory metadataFactory;
TableMetadataFactory getMetadataFactory(){
return metadataFactory;
}
void setMetadataFactory(TableMetadataFactory factory){
this.metadataFactory = factory;
}
private EngineTransactionManager transactionManager;
/**
* Gets the transactionManager.
*/
@Override
public TransactionManager getTransactionManager(){
return this.transactionManager;
}
protected EngineTransactionManager getEngineTransactionManager(){
return this.transactionManager;
}
public void setTransactionManager(EngineTransactionManager transactionManager){
this.transactionManager = transactionManager;
}
//</editor-fold>
private File directory;
protected File getDirectory(){
return directory;
}
private AtomicReference<DatabaseState> state = new AtomicReference<>();
public DatabaseState getState(){
return state.get();
}
private final Thread shutdownHook = new Thread(new Runnable(){
@Override
public void run() {
try {
XFlatDatabase.this.shutdown(1000);
} catch (TimeoutException ex) {
log.warn("Timed out while shutting down database " + directory);
}
}
});
//the engine cache
private ConcurrentHashMap<String, TableMetadata> tables = new ConcurrentHashMap<>();
private DatabaseConfig config = new DatabaseConfig();
public void setConfig(DatabaseConfig config){
if(this.state.get() != DatabaseState.Uninitialized){
throw new XFlatException("Cannot configure database after initialization");
}
this.config = config;
}
public DatabaseConfig getConfig(){
return config;
}
private Map<String, TableConfig> tableConfigs = new HashMap<>();
/**
* Configures a table with the given table configuration.
* @param tableName The name of the table to configure.
* @param config The configuration to apply.
*/
void configureTable(String tableName, TableConfig config){
if(this.state.get() != DatabaseState.Uninitialized){
throw new XFlatException("Cannot configure table after initialization");
}
this.tableConfigs.put(tableName, config);
}
private Log log = LogFactory.getLog(getClass());
private AtomicBoolean pojoConverterLoaded = new AtomicBoolean(false);
private volatile PojoConverter pojoConverter;
/**
* Gets the PojoConverter that has been used to extend the database's conversion service.
* This overrides any PojoConverter that was defined in the Database Configuration.
*/
public PojoConverter getPojoConverter(){
return pojoConverter;
}
/**
* Extends the database's conversion service with the given PojoConverter.
* This overrides any PojoConverter that was defined in the Database Configuration.
* @param converter The converter that should extend the database's conversion service.
*/
public void setPojoConverter(PojoConverter converter){
synchronized(this){
this.conversionService = converter.extend(conversionService);
this.pojoConverter = converter;
}
}
//<editor-fold desc="construction">
/**
* Creates a new database in the given directory.
* @param directory The flat-file directory in which tables should be stored.
*/
XFlatDatabase(File directory){
this(directory, null);
}
/**
* Creates a new database in the given directory.
* @param directory The flat-file directory in which tables should be stored.
* @param executorService The executor service to use for all database-related
* tasks. If null, the database will create one in Initialize.
*/
protected XFlatDatabase(File directory, ScheduledExecutorService executorService){
this.directory = directory;
this.conversionService = new DefaultConversionService();
StringConverters.registerTo(conversionService);
JDOMConverters.registerTo(conversionService);
DateConverters.registerTo(conversionService);
this.executorService = executorService;
this.metadataFactory = new TableMetadataFactory(this, new File(directory, "xflat_metadata"));
this.state = new AtomicReference<>(DatabaseState.Uninitialized);
}
/**
* Builds a new XFlatDatabase around the given file. The returned builder
* can be used to configure the database.
* @param file The directory to be managed by the XFlatDatabase.
* @return A DatabaseBuilder which constructs instances of XFlatDatabase.
*/
public static DatabaseBuilder<XFlatDatabase> Build(File file){
return new DatabaseBuilder<>(file.toURI(), new XFlatDatabaseProvider());
}
//</editor-fold>
/**
* Initializes the database. Once initialized the database can provide tables
* and operate on underlying data.
* <p/>
* The database will register a shutdown hook with the runtime to clean up any
* resources and abandon all running tasks. This shutdown hook will be removed
* when {@link #shutdown() } is called.
*/
void initialize(){
if(!this.state.compareAndSet(DatabaseState.Uninitialized, DatabaseState.Initializing)){
return;
}
try
{
String hello = String.format("Starting up XFlat database, version %s",
org.xflatdb.xflat.Version.VERSION);
if(org.xflatdb.xflat.Version.BUILD_REVISION > 0){
hello += String.format("\r\n development revision %d (%s)",
org.xflatdb.xflat.Version.BUILD_REVISION,
org.xflatdb.xflat.Version.BUILD_COMMIT.substring(0, 7));
}
log.info(hello);
if(!this.directory.exists())
this.directory.mkdirs();
this.validateConfig();
if(this.executorService == null)
this.executorService = new ScheduledThreadPoolExecutor(this.config.getThreadCount());
if(this.transactionManager == null){
this.transactionManager = new ThreadContextTransactionManager(new DocumentFileWrapper(new File(directory, "xflat_transaction")));
}
this.InitializeScheduledTasks();
Runtime.getRuntime().addShutdownHook(this.shutdownHook);
//recover transactional state if necessary
this.transactionManager.recover(this);
//done initializing
this.state.set(DatabaseState.Running);
}catch(Exception ex){
this.state.set(DatabaseState.Uninitialized);
throw new XFlatException("Initialization error", ex);
}
}
/**
* Shuts down the database.
* This method blocks until the database is completely shut down, as long
* as it takes.
*/
public void shutdown(){
try{
this.doShutdown(0);
}catch(TimeoutException ex){
throw new RuntimeException("A timeout occured that should never have happened", ex);
}
finally{
//close all resources
this.getEngineTransactionManager().close();
}
}
/**
* Shuts down the database.
* This method blocks only until the timeout expires - if the database
* is not completely shutdown in that time, a TimeoutException is thrown.
* @param timeout The number of milliseconds to wait before timing out
* @throws TimeoutException if the database did not fully shut down before
* the timeout expired.
*/
public void shutdown(int timeout) throws TimeoutException{
try{
this.doShutdown(timeout);
}
finally{
//close all resources
this.getEngineTransactionManager().close();
}
}
private void doShutdown(int timeout) throws TimeoutException{
if(!this.state.compareAndSet(DatabaseState.Running, DatabaseState.ShuttingDown)){
return;
}
if(log.isTraceEnabled())
log.trace(String.format("Shutting down, timeout %dms", timeout));
//by default, wait as long as it takes
Long lTimeout = Long.MAX_VALUE;
if(timeout > 0){
//wait only until the timeout
lTimeout = System.currentTimeMillis() + timeout;
}
//spin them all down
Set<EngineBase> engines = new HashSet<>();
for(Map.Entry<String, TableMetadata> table : this.tables.entrySet()){
try{
EngineBase e = table.getValue().spinDown(true, false);
if(e != null){
if(e.getState() == EngineState.Running){
//don't care, force spin down
e.spinDown(null);
}
engines.add(e);
}
}catch(Exception ex){
//eat
}
}
//save all metadata
for(Map.Entry<String, TableMetadata> table : this.tables.entrySet()){
try {
this.metadataFactory.saveTableMetadata(table.getValue());
} catch (IOException ex) {
this.log.warn("Unable to save metadata for table " + table.getKey(), ex);
}
}
this.tables.clear();
//wait for the engines to finish spinning down
do{
Iterator<EngineBase> it = engines.iterator();
while(it.hasNext()){
EngineBase e = it.next();
EngineState state = e.getState();
if(state == EngineState.Uninitialized ||
state == EngineState.SpunDown){
it.remove();
continue;
}
}
if(engines.isEmpty()){
//COOL! we're done
return;
}
}while(System.currentTimeMillis() < lTimeout);
//force any remaining tables to spin down now
boolean anyLeft = false;
for(EngineBase engine : engines){
anyLeft = true;
try{
if(engine != null)
engine.forceSpinDown();
}catch(Exception ex){
//eat
}
}
if(anyLeft)
throw new TimeoutException("Shutdown timed out");
try{
Runtime.getRuntime().removeShutdownHook(shutdownHook);
}catch(Exception ex){
//that's ok
}
}
private void validateConfig(){
for(Map.Entry<String, TableConfig> entry : this.tableConfigs.entrySet()){
Document existing = this.metadataFactory.getMetadataDoc(entry.getKey());
if(existing == null || existing.getRootElement() == null){
//we're good here.
continue;
}
Element cfg = existing.getRootElement().getChild("config", XFlatConstants.xFlatNs);
if(cfg == null){
//still good
continue;
}
try {
TableConfig inMetadata = TableConfig.FromElementConverter.convert(cfg);
if(!entry.getValue().equals(inMetadata)){
throw new XFlatException("Configuration for table " + entry.getKey() +
" does not match stored configuration");
}
} catch (ConversionException ex) {
//table metadata is corrupt, ignore but warn
log.warn("The metadata for table " + entry.getKey() + " is corrupt", ex);
continue;
}
}
}
private void InitializeScheduledTasks(){
this.executorService.scheduleWithFixedDelay(new Runnable(){
@Override
public void run() {
update();
}
}, 500, 500, TimeUnit.MILLISECONDS);
}
/**
* Called periodically by the executor service to perform maintenance
* on the DB.
*/
private void update(){
//check on inactivity shutdown
for(TableMetadata m : this.tables.values()){
if(m.canSpinDown()){
//spin down if no uncommitted data
m.spinDown(false, false);
}
//don't ever remove TableMetadata. It's too dangerous with the way we do locking and isn't worth it.
}
}
@Override
public <T> Table<T> getTable(Class<T> type){
return this.getTable(type, type.getSimpleName());
}
@Override
public <T> Table<T> getTable(Class<T> type, String name){
TableMetadata table = getMetadata(type, name);
TableBase ret = table.getTable(type);
return (Table<T>)ret;
}
@Override
public KeyValueTable getKeyValueTable(String name){
TableMetadata table = getMetadata(null, name);
ConvertingKeyValueTable ret = new ConvertingKeyValueTable(name);
ret.setConversionService(conversionService);
ret.setEngineProvider(table);
ret.setLoadPojoMapperAction(new Action1<ConvertingKeyValueTable>(){
@Override
public void apply(ConvertingKeyValueTable val){
try {
loadPojoConverter();
val.setConversionService(conversionService);
} catch (ClassNotFoundException | InstantiationException | IllegalAccessException ex) {
throw new UnsupportedOperationException("No conversion available between your type and Element", ex);
}
}
});
return ret;
}
/**
* Gets the internal EngineBase that has been spun up to manage the given table.
* This internal engine is the low-level manager of the database on disk.
* <p/>
* Please use {@link #getTable(java.lang.Class) } instead. It is preferable
* to interact with the data via the {@link Table} interface.
* @param name The name of the table for which an engine is desired.
* @return A running EngineBase which manages the table.
*/
public EngineBase getEngine(String name){
return getMetadata(null, name).provideEngine();
}
private TableMetadata getMetadata(Class<?> type, String name){
if(this.state.get() == DatabaseState.Uninitialized){
throw new IllegalStateException("Database has not been initialized");
}
if(this.state.get() == DatabaseState.ShuttingDown){
throw new IllegalStateException("Database is shutting down");
}
if(name == null || name.startsWith("xflat_")){
throw new IllegalArgumentException("Table name cannot be null or start with 'xflat_': " + name);
}
if(type != null && !Element.class.equals(type)){
if(!this.getConversionService().canConvert(type, Element.class) ||
!this.getConversionService().canConvert(Element.class, type)){
try {
//try to load the pojo converter
loadPojoConverter();
} catch (Exception ex) {
throw new UnsupportedOperationException("No conversion available between " +
type + " and " + Element.class, ex);
}
//check again
if(!this.getConversionService().canConvert(type, Element.class) ||
!this.getConversionService().canConvert(Element.class, type)){
throw new UnsupportedOperationException("No conversion available between " +
type + " and " + Element.class);
}
}
}
//see if we have a cached engine already
TableMetadata table = this.tables.get(name);
if(table == null){
TableConfig tblConfig = this.tableConfigs.get(name);
Class<?> idType = String.class;
if(type != null && !Element.class.equals(type)){
IdAccessor accessor = IdAccessor.forClass(type);
if(accessor != null && accessor.hasId()){
idType = accessor.getIdType();
}
}
table = this.metadataFactory.makeTableMetadata(name, new File(getDirectory(), name + ".xml"), tblConfig, idType);
TableMetadata weWereSlow = this.tables.putIfAbsent(name, table);
if(weWereSlow != null){
//this thread was slower than another thread, use the other thread's table metadata
table = weWereSlow;
}
if(log.isTraceEnabled())
log.trace(String.format("Metadata loaded for table %s", table.getName()));
}
return table;
}
private void loadPojoConverter() throws ClassNotFoundException, InstantiationException, IllegalAccessException{
if(!pojoConverterLoaded.compareAndSet(false, true)){
return;
}
if(this.pojoConverter != null){
//the user set a pojo converter via the Set method.
return;
}
Class<?> converter;
converter = this.getClass().getClassLoader().loadClass(this.config.getPojoConverterClass());
if(converter == null){
log.warn(String.format("Unable to locate Pojo Converter %s", this.config.getPojoConverterClass()));
return;
}
if(log.isTraceEnabled())
log.trace(String.format("Activating Pojo Converter %s", converter.getName()));
PojoConverter instance = (PojoConverter)converter.newInstance();
this.setPojoConverter(instance);
}
/**
* Represents the various states of the Database.
*/
public enum DatabaseState{
/**
* The state of a database before the {@link XFlatDatabase#initialize() } method is
* called.
*/
Uninitialized,
/**
* The state when the database is initializing, including potentially
* recovering from an unexpected shutdown.
*/
Initializing,
/**
* The state of a database that is running and capable of responding
* to requests.
*/
Running,
/**
* The state of a database that is either in the process of or has already
* shut down. Requests on this database will throw.
*/
ShuttingDown,
}
}