package automately.core.file;
import automately.core.data.User;
import automately.core.data.UserData;
import automately.core.file.nio.UserFilePath;
import automately.core.file.nio.UserFileSystem;
import automately.core.file.stores.AmazonS3Store;
import automately.core.file.stores.FileSystemStore;
import automately.core.services.core.AutomatelyService;
import com.hazelcast.core.*;
import com.hazelcast.map.MapInterceptor;
import com.hazelcast.map.listener.MapListener;
import io.jsync.app.core.Cluster;
import io.jsync.app.core.Logger;
import io.jsync.buffer.Buffer;
import io.jsync.json.JsonObject;
import io.jsync.utils.CryptoUtils;
import io.jsync.utils.MimeUtils;
import io.jsync.utils.Token;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import static io.jsync.utils.CryptoUtils.calculateHmacSHA512;
import static io.jsync.utils.Token.generateToken;
/**
* FileConductor is a service that is used to conduct some extra things
* for our VirtualFile API.
*/
public class VirtualFileService extends AutomatelyService implements MapInterceptor {
public static String DEFAULT_STORE = FileSystemStore.class.getCanonicalName();
/**
* By default this is null. If it is null we will store the data directly.
*/
private static VirtualFileStore fileStore = null;
public static VirtualFileStore getFileStore() {
return fileStore;
}
public static void setFileStore(VirtualFileStore fileStore) {
if (fileStore == null) {
throw new NullPointerException("The fileStore cannot be null!");
}
VirtualFileService.fileStore = fileStore;
}
private String entryListener = "";
private String mapInterceptor = "";
@Override
public void start(Cluster owner) {
Logger logger = owner.logger();
logger.info("Initializing the VirtualFileSystem.");
String store = coreConfig().getObject("file", new JsonObject()).getString("store", DEFAULT_STORE);
// This line allows us to keep support for the legacy configuration
if(store.equals("aws")){
store = AmazonS3Store.class.getCanonicalName();
}
try {
ClassLoader classLoader = getClass().getClassLoader();
Class clazz = classLoader.loadClass(store);
if(clazz != null){
if(VirtualFileStore.class.isAssignableFrom(clazz)){
logger.info("Using \"" + store + "\" as the VirtualFileStore.");
VirtualFileStore newInstance = (VirtualFileStore) clazz.newInstance();
VirtualFileService.setFileStore(newInstance);
} else {
logger.fatal("The store \"" + store + "\" is not a valid VirtualFileStore.");
}
}
} catch (Exception e){
logger.fatal(e.getMessage());
return;
}
if (fileStore == null) {
logger.fatal("No FileStore was found!");
return;
}
IMap<String, VirtualFile> files = owner.data().persistentMap("files");
mapInterceptor = files.addInterceptor(this);
fileStore.initialize(cluster());
if(fileStore instanceof MapListener && !cluster().manager().clientMode()){
entryListener = cluster().data().persistentMap("files").addEntryListener((MapListener) fileStore, true);
}
VirtualFileSystem.initialize(owner);
// This will help ensure file persistence is working
verifyNode();
// If the service is the "jobrunner" service
// we can go ahead and ignore this
if(cluster().manager().clientMode()){
return;
}
IMap<String, Object> persistentGlobalTmpData = owner.data().persistentMap("global.temp.data");
String migrationStore = coreConfig().getObject("file", new JsonObject()).getString("migration_store", store);
if(!migrationStore.equals(store) && !persistentGlobalTmpData.containsKey(migrationStore + "_migration")){
logger.info("Attempting to migrate file data from the file store \"" + migrationStore + "\".");
try {
ClassLoader classLoader = getClass().getClassLoader();
Class clazz = classLoader.loadClass(migrationStore);
if(clazz != null){
if(VirtualFileStore.class.isAssignableFrom(clazz)){
logger.info("Using \"" + migrationStore + "\" as the migration VirtualFileStore.");
VirtualFileStore newInstance = (VirtualFileStore) clazz.newInstance();
if(newInstance != null){
newInstance.initialize(cluster());
logger.info("Migrating file data using the store \"" + migrationStore + "\".");
for (VirtualFile file : files.values()) {
Buffer data = newInstance.readRawData(file);
if(data != null){
logger.debug("Migrating file " + file.token());
file.size = data.length();
fileStore.writeRawData(file, data);
files.set(file.token(), file);
}
}
logger.info("File data migration finished.");
JsonObject fileConfig = coreConfig().getObject("file", new JsonObject());
fileConfig.removeField("migration_store");
coreConfig().putObject("file", fileConfig);
cluster().config().save();
// We can store this value for 24 hours. This is in case migrations aren't ran
persistentGlobalTmpData.put(migrationStore + "_migration", true, 24, TimeUnit.HOURS);
}
} else {
logger.fatal("The store \"" + store + "\" is not a valid migration VirtualFileStore.");
}
}
} catch (Exception e){
logger.fatal("File data migration error (" + e + ")");
}
}
}
private void verifyNode(){
HazelcastInstance hazelcast = cluster().hazelcast();
Set<Member> members = hazelcast.getCluster().getMembers();
Logger logger = cluster().logger();
logger.info("Starting Virtual File System node verification...");
User adminUser = UserData.getUserByUsername("admin");
UserFileSystem adminFs = VirtualFileSystem.getUserFileSystem(adminUser);
// If this file is deleted it will cause errors within the system.
UserFilePath path = adminFs.getPath("/.node_verification");
if(members.size() == 1 && !cluster().manager().clientMode()){
logger.info("Triggering master node verification file creation...");
// TODO this may be causing blocking issues..
// Let's go ahead and create
// the initial verification file
if(!Files.exists(path)){
try {
Files.createFile(path);
} catch (IOException e) {
throw new RuntimeException("Could not start the VirtualFileService. (Node verification failure)", e);
}
}
try {
VirtualFile virtualFile = adminFs.getFile(path);
String hashToken = calculateHmacSHA512(generateToken().toString(), virtualFile.token());
ILock fileLock = cluster().hazelcast().getLock("fs.file.lock." + virtualFile.token());
fileLock.forceUnlock();
Files.write(path, hashToken.getBytes());
// We can simply lock this as long as the node is running
fileLock.lock();
// We can now store the verification data
} catch (IOException e) {
throw new RuntimeException("Could not start the VirtualFileService. (Node verification failure)", e);
}
logger.info("Verification file created...");
}
if(!Files.exists(path)){
throw new RuntimeException("Could not start the VirtualFileService. (Node verification failure - \"/.node_verification\" does not exist)");
}
try {
VirtualFile virtualFile = adminFs.getFile(path);
byte[] fileData = Files.readAllBytes(path);
logger.info("Verifying node via verification file...");
if(!fileStore.verifyNode(virtualFile, new String(fileData))){
throw new RuntimeException("Node file verification failed!");
}
} catch (IOException e) {
throw new RuntimeException("Could not start the VirtualFileService. (Node verification failure)", e);
}
logger.info("Virtual File System node verification success!");
}
@Override
public void stop() {
IMap files = cluster().data().persistentMap("files");
if(!mapInterceptor.isEmpty()){
files.removeInterceptor(mapInterceptor);
}
if(fileStore instanceof MapListener && !entryListener.isEmpty()){
files.removeEntryListener(entryListener);
}
try {
fileStore.stop();
} catch (Exception ignored){
}
}
@Override
public String name() {
return getClass().getCanonicalName();
}
@Override
public Object interceptGet(Object o) {
// We can ignore this operation
return null;
}
@Override
public void afterGet(Object o) {
// We can ignore this operation
}
@Override
public Object interceptPut(Object oldValue, Object newValue) {
if(newValue instanceof VirtualFile){
VirtualFile file = (VirtualFile) newValue;
if(!file.pathAlias.endsWith("/")){
file.pathAlias += "/";
}
if(file.name.contains("/")){
throw new RuntimeException("name cannot contain /");
}
if(file.userToken == null || file.userToken.isEmpty()){
throw new RuntimeException("userToken cannot be empty");
}
if (file.pathAlias == null || file.pathAlias.isEmpty()) {
file.pathAlias = "/";
}
if (file.type == null || file.type.isEmpty()) {
/**
* Automatically get and retrieve the mime-type
*/
String fRegex = "\\.(?=[^\\\\.]+$)";
if (file.name.split(fRegex).length > 1) {
if (MimeUtils.getMimeTypeForExtension(file.name.split(fRegex)[1]) != null) {
String extension = file.name.split(fRegex)[1];
file.type = MimeUtils.getMimeTypeForExtension(extension);
}
}
if(file.type == null || file.type.isEmpty()){
file.type = "application/octet-stream";
}
}
return file;
}
return null;
}
@Override
public void afterPut(Object o) {
// We can ignore this operation
}
@Override
public Object interceptRemove(Object o) {
// We can ignore this operation
return null;
}
@Override
public void afterRemove(Object o) {
// We can ignore this operation
}
}