package automately.core.services.http;
import automately.core.data.Meta;
import automately.core.data.User;
import automately.core.file.VirtualFile;
import automately.core.file.VirtualFileService;
import automately.core.file.nio.UserFilePath;
import automately.core.file.nio.UserFileSystem;
import automately.core.services.core.AutomatelyService;
import com.google.common.net.InternetDomainName;
import com.hazelcast.core.EntryEvent;
import com.hazelcast.core.IMap;
import com.hazelcast.map.listener.EntryEvictedListener;
import com.hazelcast.map.listener.EntryRemovedListener;
import com.hazelcast.map.listener.EntryUpdatedListener;
import io.jsync.Async;
import io.jsync.Handler;
import io.jsync.app.core.Cluster;
import io.jsync.app.core.Config;
import io.jsync.app.core.Logger;
import io.jsync.buffer.Buffer;
import io.jsync.dns.DnsClient;
import io.jsync.http.*;
import io.jsync.impl.ConcurrentHashSet;
import io.jsync.json.JsonArray;
import io.jsync.json.JsonElement;
import io.jsync.json.JsonObject;
import io.jsync.streams.Pump;
import io.jsync.utils.CryptoUtils;
import org.apache.http.client.utils.DateUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.zip.CRC32;
import static automately.core.data.UserData.getMeta;
import static automately.core.data.UserData.getUserByToken;
import static automately.core.file.VirtualFileSystem.getUserFileSystem;
import static automately.core.file.VirtualFileSystem.readFileData;
@SuppressWarnings("Duplicates")
/**
* The ClusteredHttpServer is a powerful distributed HttpServer that
* allows certain modules to handle requests for hosts registered in the cluster.
*/
public class ClusteredHttpServer extends AutomatelyService implements EntryUpdatedListener<String, JsonElement>, EntryRemovedListener<String, JsonElement>, EntryEvictedListener<String, JsonElement> {
// This is the Default. You must change this manually when implementing the HttpServerObject
// Obviously you cannot use the automate.ly domain
public static String DEFAULT_HOST_PREFIX = "alias.clustered.http.automate.ly";
public static String DEFAULT_SERVER_NAME = "automately";
public static boolean HOST_VALIDATION_DISABLED = false;
public static String DEFAULT_OFFLINE_PATH = "/public/www/default/";
public static String DEFAULT_404_MESSAGE = "The requested resource was not found.";
public static int DEFAULT_SERVER_RECEIVE_BUFFER_SIZE = 4096 * 2;
public static int DEFAULT_SERVER_SEND_BUFFER_SIZE = 4096 * 2;
// This is set to a high number by default because Automately
// is designed for systems tuned for maximum performance
public static int DEFAULT_SERVER_ACCEPT_BACKLOG_SIZE = 20000;
// This is for the HttpServerObject and the client connecting to that server - TODO look into this
public static int DEFAULT_CLIENT_RECEIVE_BUFFER_SIZE = 4096 * 2;
public static int DEFAULT_CLIENT_SEND_BUFFER_SIZE = 4096 * 2;
private static boolean firstLoadComplete = false;
private static Cluster cluster;
// We create a local async so we can
// ensure we don't utilize io on the main one
private Async async;
private HttpServer httpServer;
private Map<String, HttpClient> httpClients = new ConcurrentHashMap<>();
private int httpClientReceiveBufferSize = DEFAULT_CLIENT_RECEIVE_BUFFER_SIZE;
private int httpClientSendBufferSize = DEFAULT_CLIENT_SEND_BUFFER_SIZE;
private Logger logger;
private IMap<String, JsonArray> clusteredHosts;
private Map<String, JsonArray> cachedClusteredHosts;
private IMap<String, JsonObject> reservedHostData;
private Map<String, JsonObject> cachedReservedHostData;
private static void checkInitialized(){
if (!firstLoadComplete || cluster == null) {
throw new RuntimeException("The server has not been initialized yet!");
}
}
public static String getServerHost(){
checkInitialized();
Config config = cluster.config();
JsonObject rawConfig = config.rawConfig();
JsonObject coreConfig = rawConfig.getObject("automately", new JsonObject()).getObject("core", new JsonObject());
JsonObject httpConfig = coreConfig.getObject("clustered_http_server", new JsonObject());
return httpConfig.getString("host", config.clusterHost());
}
/**
* This will retieve an array of all the currently allowed host_prefixes on this machine
*
* @return
*/
public static List<String> getHostPrefixes() {
checkInitialized();
JsonObject rawConfig = cluster.config().rawConfig();
if (!rawConfig.containsField("automately")) {
rawConfig.putValue("automately", new JsonObject().putValue("core", new JsonObject()));
}
JsonObject coreConfig = rawConfig.getObject("automately").getObject("core");
JsonObject httpConfig = coreConfig.getObject("clustered_http_server", new JsonObject());
// Default value is .alias.clustered.http.automate.ly
JsonArray hostPrefixes = httpConfig.getArray("host_prefixes",
new JsonArray().add(DEFAULT_HOST_PREFIX));
LinkedList<String> converted = new LinkedList<>();
for (Object value : hostPrefixes) {
if (value instanceof String) {
converted.add(value.toString().trim());
}
}
return converted;
}
public static int getClientReceiveBufferSize(){
checkInitialized();
JsonObject rawConfig = cluster.config().rawConfig();
if (!rawConfig.containsField("automately")) {
rawConfig.putValue("automately", new JsonObject().putValue("core", new JsonObject()));
}
JsonObject coreConfig = rawConfig.getObject("automately").getObject("core");
JsonObject httpConfig = coreConfig.getObject("clustered_http_server", new JsonObject());
return httpConfig.getInteger("client_receive_buffer_size", DEFAULT_CLIENT_RECEIVE_BUFFER_SIZE);
}
public static int getClientSendBufferSize(){
checkInitialized();
JsonObject rawConfig = cluster.config().rawConfig();
if (!rawConfig.containsField("automately")) {
rawConfig.putValue("automately", new JsonObject().putValue("core", new JsonObject()));
}
JsonObject coreConfig = rawConfig.getObject("automately").getObject("core");
JsonObject httpConfig = coreConfig.getObject("clustered_http_server", new JsonObject());
return httpConfig.getInteger("client_send_buffer_size", DEFAULT_CLIENT_SEND_BUFFER_SIZE);
}
public static boolean usingHostPrefix(String host) {
List<String> prefixes = getHostPrefixes();
for (String prefix : prefixes) {
if (host.endsWith(prefix)) {
return true;
}
}
return false;
}
@Override
public void start(Cluster owner) {
cluster = owner;
async = cluster.async();
logger = cluster.logger();
Config config = cluster.config();
// We must always set this
firstLoadComplete = true;
if ((!config.isRole("api") && !config.isRole("sdk") && !config.isAll()) || cluster().manager().clientMode()) {
logger.info("Not starting the Clustered HttpServer.");
return;
}
JsonObject coreConfig = coreConfig();
JsonObject httpConfig = coreConfig.getObject("clustered_http_server", new JsonObject());
int serverPort = httpConfig.getInteger("port", 9888);
String serverHost = httpConfig.getString("host", config.clusterHost());
boolean disableHostValidation = httpConfig.getBoolean("disable_host_validation",
ClusteredHttpServer.HOST_VALIDATION_DISABLED);
JsonArray hostPrefixes = httpConfig.getArray("host_prefixes",
new JsonArray().add(DEFAULT_HOST_PREFIX));
int sendBufferSize = httpConfig.getInteger("send_buffer_size", DEFAULT_SERVER_SEND_BUFFER_SIZE);
int receiveBufferSize = httpConfig.getInteger("receive_buffer_size", DEFAULT_SERVER_RECEIVE_BUFFER_SIZE);
int acceptBacklog = httpConfig.getInteger("accept_backlog", DEFAULT_SERVER_ACCEPT_BACKLOG_SIZE);
httpClientSendBufferSize = httpConfig.getInteger("client_send_buffer_size", DEFAULT_CLIENT_SEND_BUFFER_SIZE);
httpClientReceiveBufferSize = httpConfig.getInteger("client_receive_buffer_size", DEFAULT_CLIENT_RECEIVE_BUFFER_SIZE);
logger.info("Using the Host Prefix(s): " + hostPrefixes.toString());
httpConfig.putNumber("port", serverPort);
httpConfig.putString("host", serverHost);
httpConfig.putArray("host_prefixes", hostPrefixes);
httpConfig.putBoolean("disable_host_validation", disableHostValidation);
ClusteredHttpServer.HOST_VALIDATION_DISABLED = disableHostValidation;
coreConfig().putObject("clustered_http_server", httpConfig);
cluster.config().save();
if (disableHostValidation) {
logger.warn("Host validation is disabled.");
}
httpServer = async.createHttpServer();
cachedClusteredHosts = new ConcurrentHashMap<>();
cachedReservedHostData = new ConcurrentHashMap<>();
Set<String> cachedValidatedHosts = new ConcurrentHashSet<>();
clusteredHosts = cluster.data().getMap("http.clustered.hosts");
clusteredHosts.addEntryListener(this, true);
reservedHostData = cluster.data().persistentMap("http.clustered.hosts.reserved.data");
reservedHostData.addEntryListener(this, true);
logger.info("Populating initial cached data...");
cachedClusteredHosts.putAll(clusteredHosts);
cachedReservedHostData.putAll(reservedHostData);
logger.info("Finished populating initial cached data...");
// Performance tuning options
httpServer.setSendBufferSize(sendBufferSize);
httpServer.setReceiveBufferSize(receiveBufferSize);
httpServer.setAcceptBacklog(acceptBacklog);
// The ErrorCount is a simple map that we can use to see which backend
// servers have the most errors.
Map<String, Integer> errorCount = new ConcurrentHashMap<>();
httpServer.requestHandler(new Handler<HttpServerRequest>() {
private Handler<HttpServerRequest> requestHandler = this;
private void handleProxiedResponse(HttpServerRequest incomingRequest, String hostIp, int hostPort) {
String hostId = hostIp + ":" + hostPort;
HttpClient httpClient = httpClients.get(hostId);
if(httpClient == null){
httpClient = async.createHttpClient();
httpClient.setSendBufferSize(httpClientSendBufferSize);
httpClient.setReceiveBufferSize(httpClientReceiveBufferSize);
// This can be adjusted as needed. Remember this create's an http client
// per hostIp and hostPort
httpClient.setMaxPoolSize(64 * Runtime.getRuntime().availableProcessors());
httpClient.setKeepAlive(true);
httpClient.setHost(hostIp);
httpClient.setPort(hostPort);
httpClients.put(hostId, httpClient);
}
HttpServerResponse proxyResponse = incomingRequest.response();
proxyResponse.setChunked(true);
HttpClientRequest outgoingRequest = httpClient.request(incomingRequest.method(), incomingRequest.uri(), response -> {
proxyResponse.setStatusCode(response.statusCode());
proxyResponse.setStatusMessage(response.statusMessage());
response.headers().forEach(header -> proxyResponse.putHeader(header.getKey(), header.getValue()));
Pump pump = Pump.createPump(response, proxyResponse);
response.endHandler(voidz -> {
// pump.stop();
proxyResponse.end();
});
pump.start();
});
outgoingRequest.exceptionHandler(event -> {
event.printStackTrace();
int newCount = errorCount.getOrDefault(hostId, 0);
newCount++;
errorCount.put(hostId, newCount);
String host1 = incomingRequest.headers().get("Host");
if (host1 == null) {
host1 = "";
}
if (host1.split(":").length > 0) {
host1 = host1.split(":")[0];
}
host1 = host1.trim();
logger.error("Error on the host " + host1 + " with the hostId \"" + hostId + "\"...");
// We can handle the response offline...
JsonObject reservedHost = cachedReservedHostData.get(host1);
if(reservedHost != null){
// Let's attempt to handle the request offline.
if(handleOfflineResponse(incomingRequest, reservedHost)){
// If there has been more than 5 errors in the response
// we are going to simply remove it from the handlers
// TODO add config option for max_backend_errors
if(newCount > 5){
logger.info("Error threshold reached for the host " + host1 + " with the hostId " + hostId);
JsonArray newArr = new JsonArray();
for (Object obj : cachedClusteredHosts.get(host1)) {
if(obj.toString().endsWith(hostId)){
continue;
}
newArr.add(obj);
}
if(newArr.size() > 0){
// Note: it is important that we do not update cachedClusteredHosts but simply clusteredHosts
clusteredHosts.put(host1, newArr);
// We need to respond with this one now..
String handlerId = newArr.size() > 0 ? newArr.get(0) : null;
// TODO block invalid ports..
// TODO copy this into it's own... TODO decode what I even meant..
if(handlerId != null){
if(handlerId.indexOf("proxy.direct:") == 0){
String proxyHost = handlerId.split(":")[1];
String proxyPort = handlerId.split(":")[2];
handleProxiedResponse(incomingRequest, proxyHost, Integer.valueOf(proxyPort));
return;
}
}
} else {
clusteredHosts.remove(host1);
}
}
return;
}
}
proxyResponse.setStatusMessage("Service Unreachable");
proxyResponse.setStatusCode(503);
proxyResponse.end("The connection to the backend server was refused.");
});
outgoingRequest.setChunked(true);
if(!incomingRequest.headers().contains("X-Forwarded-For")){
outgoingRequest.putHeader("X-Forwarded-For", incomingRequest.remoteAddress().getHostString());
}
if(!incomingRequest.headers().contains("X-Forwarded-Host")){
outgoingRequest.putHeader("X-Forwarded-Host", incomingRequest.headers().get("Host"));
}
incomingRequest.headers().forEach(header -> {
// We need to make sure we store the Content-Length properly
if(header.getKey().equals("Content-Length")){
outgoingRequest.putHeader("X-Content-Length", header.getValue());
} else {
outgoingRequest.putHeader(header.getKey(), header.getValue());
}
});
// README : add custom headers here if you need to
Pump.createPump(incomingRequest, outgoingRequest).start();
incomingRequest.endHandler(voidz -> {
// pump.stop();
outgoingRequest.end();
});
}
private boolean handleOfflineResponse(HttpServerRequest request, JsonObject hostData){
String userToken = hostData.getString("user");
User user = getUserByToken(userToken);
String offlinePath;
if(hostData.containsField("offlinePath")){
offlinePath = hostData.getString("offlinePath");
} else {
Meta defaultOfflinePath = getMeta(user, "defaultOfflinePath");
if(defaultOfflinePath != null){
offlinePath = defaultOfflinePath.value.toString();
} else {
offlinePath = DEFAULT_OFFLINE_PATH;
}
}
if(offlinePath != null) {
// We call runOnContext so we don't take up time with fs stuff
async.runOnContext(event -> {
UserFileSystem fs = getUserFileSystem(user);
UserFilePath realPath = fs.getPath(offlinePath);
String requestPath = request.path();
if (requestPath == null || requestPath.equals("")) {
requestPath = "/";
}
// Automatically handle index.html if the path is empty
if (requestPath.equals("/") || requestPath.endsWith("/")) {
requestPath = "/index.html";
}
if (requestPath.startsWith("/")) {
requestPath = requestPath.substring(1);
}
UserFilePath newPath = realPath.resolve(requestPath);
UserFilePath newHtmlPath = fs.getPath(newPath.toString() + ".html");
VirtualFile file = null;
try {
file = fs.getFile(newPath);
if (file.isDirectory) {
try {
file = fs.getFile(newHtmlPath);
} catch (FileNotFoundException ignored2) {
}
}
} catch (IOException ignored) {
try {
file = fs.getFile(newHtmlPath);
} catch (IOException ignored2) {
}
}
if (file != null) {
handleFileResponse(request, file);
return;
}
HttpServerResponse response = request.response();
response.setStatusCode(404);
response.setStatusMessage("Not Found");
response.setContentType("text/html");
response.end(DEFAULT_404_MESSAGE);
});
return true;
}
return false;
}
private void handleFileResponse(HttpServerRequest request, VirtualFile file) {
HttpServerResponse response = request.response();
response.putHeader("Cache-Control", "cache-control: private, max-age=0, no-cache");
Buffer fileData;
try {
File realFile = VirtualFileService.getFileStore().toFile(file);
if (realFile != null) {
response.setContentType(file.type);
response.sendFile(realFile.getPath());
return;
}
} catch (UnsupportedOperationException ignored) {
}
fileData = readFileData(file);
response.setContentLength(fileData.length());
response.setContentType(file.type);
response.end(fileData);
}
@Override
public void handle(HttpServerRequest req) {
/**
* Begin default headers.
*/
req.response().putHeader("Server", DEFAULT_SERVER_NAME);
req.response().putHeader("Date", DateUtils.formatDate(new Date(System.currentTimeMillis())));
req.response().setContentType("text/html");
String host1 = req.headers().get("Host");
if (host1 == null) {
host1 = "";
}
if (host1.split(":").length > 0) {
host1 = host1.split(":")[0];
}
host1 = host1.trim();
// Important this is an attempt to speed up host validation
// so we are not checking every single request
if(!cachedValidatedHosts.contains(host1)){
final String finalHost = host1;
if(!cluster().config().isDebug() && !usingHostPrefix(host1) && !disableHostValidation){
JsonObject reservedHost = cachedReservedHostData.get(host1);
logger.info("Attempting to validate the host \"" + host1 + "\".");
if(reservedHost != null && !reservedHost.getBoolean("validated", false)){
logger.info("The host \"" + host1 + "\" is reserved and needs to be validated.");
// We need to make sure we don't block anything doing
// activation
async.runOnContext(event -> {
logger.info("Running host validation on \"" + finalHost + "\".");
if (InternetDomainName.isValid(finalHost)) {
String tld = InternetDomainName.from(finalHost).topPrivateDomain().toString(); // We want to get the TLD
JsonObject hostData = cachedReservedHostData.get(finalHost);
String hostValidationToken = hostData.getString("token");
String userToken = hostData.getString("user");
User user = users().get(userToken);
if (user != null) {
if(!user.enabled){
logger.warn("The user \"" + user.username + "\" is currently disabled.");
// It will timeout throwing error
logger.info("The host \"" + finalHost + "\" could not be validated.");
req.response().setStatusCode(500);
req.response().end("Host Validation Failed");
return;
}
logger.info("Attempting to validate the TLD \"" + tld + "\" for the host \"" + finalHost + "\" with the user \"" + user.username + "\".");
Buffer tokenBuff = new Buffer();
tokenBuff.appendString(CryptoUtils.calculateHmacSHA1(
hostValidationToken, user.token() + hostValidationToken));
CRC32 crc32 = new CRC32();
crc32.update(tokenBuff.getBytes());
// Use the TLD and host to generate the proper validation prefix
String validationHost = "val" + String.format("%x", crc32.getValue()) + "." + tld;
InetSocketAddress address = new InetSocketAddress("8.8.4.4", 53);
DnsClient dnsClient = async.createDnsClient(address);
dnsClient.resolveTXT(validationHost, event1 -> {
if (event1.succeeded()) {
List<String> results = event1.result();
for (String record : results) {
if (record.trim().equals(hostValidationToken)) {
hostData.putBoolean("validated", true);
// It is important that we update this and not cachedReservedHostData
reservedHostData.set(finalHost, hostData);
logger.info("The host \"" + finalHost + "\" has been validated.");
cachedValidatedHosts.add(finalHost);
requestHandler.handle(req);
return;
}
}
}
// It will timeout throwing error
logger.info("The host \"" + finalHost + "\" could not be validated.");
req.response().setStatusCode(500);
req.response().end("Host Validation Failed");
});
}
}
});
return;
}
}
cachedValidatedHosts.add(finalHost);
async.setTimer((1000 * 60 * 60 * 24), event -> cachedValidatedHosts.remove(finalHost));
}
// Let's choose a host at random
// TODO getOrDefault is not the best..
Object[] handlerArr = cachedClusteredHosts.getOrDefault(host1, clusteredHosts.getOrDefault(host1, new JsonArray())).toArray();
if(handlerArr.length > 0 && !cachedClusteredHosts.containsKey(host1)){
cachedClusteredHosts.put(host1, new JsonArray(handlerArr));
}
List<String> newHandlerArr = Arrays.asList(Arrays.copyOf(handlerArr, handlerArr.length, String[].class));
Collections.shuffle(newHandlerArr);
String handlerId = newHandlerArr.size() > 0 ? newHandlerArr.get(0) : null;
if(handlerId != null){
if(handlerId.indexOf("proxy.direct:") == 0){
String proxyHost = handlerId.split(":")[1];
String proxyPort = handlerId.split(":")[2];
handleProxiedResponse(req, proxyHost, Integer.valueOf(proxyPort));
return;
}
}
JsonObject reservedHost = cachedReservedHostData.get(host1);
if(reservedHost != null){
// Let's attempt to handle the request offline.
if(handleOfflineResponse(req, reservedHost)){
return;
}
}
logger.info("No server found for the host \"" + host1 + "\".");
// We do a 404 by default so we can let the http request know that we don't know how to respond to it
req.response().setStatusCode(503);
req.response().setStatusMessage("No Server Found");
req.response().end("No Server Found");
}
}).listen(serverPort, serverHost, event -> {
if (event.succeeded()) {
logger.info("Started the HttpServer for the ClusteredHttpServer on " + serverHost + ":" + serverPort);
logger.info("The ClusteredHttpServer is ready to start routing");
} else {
logger.error("Failed to start the HttpServer for the ClusteredHttpServer on " + serverHost + ":" + serverPort);
}
});
}
@Override
public void stop() {
if (httpServer != null) {
logger.info("Stopping the HttpServer for the ClusteredHttpServer");
httpServer.close();
}
}
@Override
public String name() {
return getClass().getCanonicalName();
}
@Override
public void entryEvicted(EntryEvent<String, JsonElement> entry) {
entryRemoved(entry);
}
@Override
public void entryRemoved(EntryEvent<String, JsonElement> entry) {
String key = entry.getKey();
if(entry.getName().equals(clusteredHosts.getName()) && cachedClusteredHosts.containsKey(key)){
for (Object val : cachedClusteredHosts.get(key)) {
if(val instanceof String){
if(((String) val).startsWith("proxy.direct:")){
String httpKey = ((String) val).replace("proxy.direct:", "");
HttpClient client = httpClients.get(httpKey);
if(client != null){
try {
client.close();
} catch (Exception ignored){}
httpClients.remove(httpKey);
}
}
}
}
cachedClusteredHosts.remove(key);
} else if(entry.getName().equals(reservedHostData.getName()) && cachedReservedHostData.containsKey(key)){
cachedReservedHostData.remove(key);
}
}
@Override
public void entryUpdated(EntryEvent<String, JsonElement> entry) {
String key = entry.getKey();
JsonElement value = entry.getValue();
if(entry.getName().equals(clusteredHosts.getName()) && cachedClusteredHosts.containsKey(key) && value.isArray()){
cachedClusteredHosts.put(key, value.asArray());
} else if(entry.getName().equals(reservedHostData.getName()) && cachedReservedHostData.containsKey(key) && value.isObject()){
cachedReservedHostData.put(key, value.asObject());
}
}
}