/**
* This file is part of Daxplore Presenter.
*
* Daxplore Presenter is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 2.1 of the License, or
* (at your option) any later version.
*
* Daxplore Presenter is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Daxplore Presenter. If not, see <http://www.gnu.org/licenses/>.
*/
package org.daxplore.presenter.server.servlets;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.jdo.PersistenceManager;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.fileupload.util.Streams;
import org.apache.commons.io.IOUtils;
import org.daxplore.presenter.server.ServerTools;
import org.daxplore.presenter.server.admin.UploadFileManifest;
import org.daxplore.presenter.server.storage.DeleteData;
import org.daxplore.presenter.server.storage.LocaleStore;
import org.daxplore.presenter.server.storage.PMF;
import org.daxplore.presenter.server.storage.PrefixStore;
import org.daxplore.presenter.server.storage.SettingItemStore;
import org.daxplore.presenter.server.storage.StatDataItemStore;
import org.daxplore.presenter.server.storage.TextFileStore;
import org.daxplore.presenter.server.throwable.BadRequestException;
import org.daxplore.presenter.server.throwable.InternalServerException;
import org.daxplore.presenter.shared.SharedTools;
import org.daxplore.shared.SharedResourceTools;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
import com.google.apphosting.api.DeadlineExceededException;
/**
* A servlet for uploading data to the Daxplore Presenter.
*
* <p>Accepts a single zip-file that contains all the user-specific data
* that the Presenter will ever use. This includes texts, the statistical
* data, settings and possibly other data like images.</p>
*
* <p>An upload file can be generated using the Daxplore Producer project.</p>
*
* <p>Only accessible by administrators.</p>
*/
@SuppressWarnings("serial")
public class AdminUploadServlet extends HttpServlet {
private static Logger logger = Logger.getLogger(AdminUploadServlet.class.getName());
@Override
public void doPost(HttpServletRequest request, HttpServletResponse response) {
try {
long time = System.nanoTime();
int statusCode = HttpServletResponse.SC_OK;
response.setContentType("text/html; charset=UTF-8");
ServletFileUpload upload = new ServletFileUpload();
PersistenceManager pm = null;
String prefix = null;
try {
FileItemIterator fileIterator = upload.getItemIterator(request);
String fileName = "";
byte[] fileData = null;
while(fileIterator.hasNext()) {
FileItemStream item = fileIterator.next();
try (InputStream stream = item.openStream()) {
if(item.isFormField()) {
if(item.getFieldName().equals("prefix")) {
prefix = Streams.asString(stream);
} else {
throw new BadRequestException("Form contains extra fields");
}
} else {
fileName = item.getName();
fileData = IOUtils.toByteArray(stream);
}
}
}
if(SharedResourceTools.isSyntacticallyValidPrefix(prefix)) {
if(fileData!=null && !fileName.equals("")) {
pm = PMF.get().getPersistenceManager();
unzipAll(pm, prefix, fileData);
} else {
throw new BadRequestException("No file uploaded");
}
} else {
throw new BadRequestException("Request made with invalid prefix: '" + prefix + "'");
}
logger.log(Level.INFO, "Unpacked new data for prefix '" + prefix + "' in " + ((System.nanoTime()-time)/1000000000.0) + " seconds");
} catch (FileUploadException | IOException | BadRequestException e) {
logger.log(Level.WARNING, e.getMessage(), e);
statusCode = HttpServletResponse.SC_BAD_REQUEST;
} catch (InternalServerException e) {
logger.log(Level.SEVERE, e.getMessage(), e);
statusCode = HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
} catch (DeadlineExceededException e) {
logger.log(Level.SEVERE, "Timeout when uploading new data for prefix '" + prefix + "'", e);
// the server is currently unavailable because it is overloaded (hopefully)
statusCode = HttpServletResponse.SC_SERVICE_UNAVAILABLE;
} finally {
if(pm!=null) {
pm.close();
}
}
response.setStatus(statusCode);
try (PrintWriter resWriter = response.getWriter()) {
if(resWriter != null) {
resWriter.write(Integer.toString(statusCode));
resWriter.close();
}
}
} catch (IOException | RuntimeException e) {
logger.log(Level.SEVERE, "Unexpected exception: " + e.getMessage(), e);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}
private static void unzipAll(PersistenceManager pm, String prefix, byte[] fileData) throws BadRequestException, InternalServerException {
LinkedHashMap<String, byte[]> fileMap = new LinkedHashMap<>();
try(ZipInputStream zipIn = ServerTools.getAsZipInputStream(fileData)) {
// Unzip all the files and put them in a map
try {
ZipEntry entry;
while ((entry = zipIn.getNextEntry()) != null) {
if (!entry.isDirectory()) {
byte[] data = IOUtils.toByteArray(zipIn);
fileMap.put(entry.getName(), data);
}
}
} catch (IOException e) {
throw new BadRequestException("Error when reading uploaded file (invalid file?)", e);
}
// Read the file manifest to get metadata about the file
if (!fileMap.containsKey("manifest.xml")) {
throw new BadRequestException("No manifest.xml found in uploaded file");
}
try (InputStream manifestStream = new ByteArrayInputStream(fileMap.get("manifest.xml"))) {
UploadFileManifest manifest = new UploadFileManifest(manifestStream);
// Check manifest content and make sure that the file is in proper order
if (!ServerTools.isSupportedUploadFileVersion(manifest.getVersionMajor(), manifest.getVersionMinor())) {
throw new BadRequestException("Unsupported file version");
}
Locale unsupportedLocale = null;
for (Locale locale : manifest.getSupportedLocales()) {
if (!ServerTools.isSupportedLocale(locale)) {
unsupportedLocale = locale;
break;
}
}
if(unsupportedLocale!=null) {
//TODO move exception into the for-loop above if Eclipse/Java stops warning about it
throw new BadRequestException("Unsupported language: " + unsupportedLocale.toLanguageTag());
}
Set<String> missingUploadFiles = SharedResourceTools.findMissingUploadFiles(fileMap.keySet(),
manifest.getSupportedLocales());
if (!missingUploadFiles.isEmpty()) {
throw new BadRequestException("Uploaded doesn't contain required files: "
+ SharedTools.join(missingUploadFiles, ", "));
}
Set<String> unwantedUploadFiles = SharedResourceTools.findUnwantedUploadFiles(fileMap.keySet(),
manifest.getSupportedLocales());
if (!unwantedUploadFiles.isEmpty()) {
throw new BadRequestException("Uploaded file contains extra files: "
+ SharedTools.join(unwantedUploadFiles, ", "));
}
// Purge all existing data that uses this prefix, but save gaID
String gaID = SettingItemStore.getProperty(pm, prefix, "adminpanel", "gaID");
String statStoreKey = prefix + "#adminpanel/gaID";
String deleteResult = DeleteData.deleteForPrefix(pm, prefix);
pm.makePersistent(new SettingItemStore(statStoreKey, gaID));
logger.log(Level.INFO, deleteResult);
// Since we just deleted the prefix and all it's data, we have to add it
// again
pm.makePersistent(new PrefixStore(prefix));
logger.log(Level.INFO, "Added prefix to system: '" + prefix + "'");
LocaleStore localeStore = new LocaleStore(prefix, manifest.getSupportedLocales(), manifest.getDefaultLocale());
pm.makePersistent(localeStore);
logger.log(Level.INFO, "Added locale settings for prefix '" + prefix + "'");
for (String fileName : fileMap.keySet()) {
String storeName = prefix + "#" + fileName;
if (fileName.startsWith("data")) {
unpackStatisticalDataFile(pm, prefix, fileMap.get(fileName));
} else if (fileName.startsWith("groups") || fileName.startsWith("perspectives")
|| fileName.startsWith("questions") || fileName.startsWith("boolsettings")) {
unpackStaticFile(pm, storeName, fileMap.get(fileName));
} else if(fileName.startsWith("usertexts")) {
unpackPropertyFile(pm, storeName, fileMap.get(fileName));
unpackStaticFile(pm, storeName, fileMap.get(fileName));
}
}
} catch (BadRequestException e) {
throw e;
}
} catch (IOException e) {
throw new InternalServerException("Failed to close unzip file", e);
}
}
private static void unpackPropertyFile(PersistenceManager pm, String fileName, byte[] fileData) throws InternalServerException {
int addedSettings = 0;
try(BufferedReader reader = ServerTools.getAsBufferedReader(fileData)) {
List<SettingItemStore> items = new LinkedList<>();
JSONObject dataMap = (JSONObject) JSONValue.parse(reader);
for (Object prop : dataMap.keySet()) {
String key = fileName.substring(0, fileName.lastIndexOf('.')) + "/" + prop;
String value = (String) dataMap.get(prop);
items.add(new SettingItemStore(key, value));
addedSettings++;
}
pm.makePersistentAll(items);
logger.log(Level.INFO, "Set " + addedSettings + " properties from the file '" + fileName + "'");
//TODO tell uploading user of missing properties
} catch(IOException e) {
throw new InternalServerException("Failed to close unpack property file", e);
}
}
private static void unpackStaticFile(PersistenceManager pm, String fileName, byte[] fileData) throws InternalServerException {
TextFileStore item;
try {
item = new TextFileStore(fileName, new String(fileData, "UTF-8"));
pm.makePersistent(item);
logger.log(Level.INFO, "Stored the static file '" + fileName + "'");
} catch (UnsupportedEncodingException e) {
throw new InternalServerException("UTF-8 not supported by server", e);
}
}
private static void unpackStatisticalDataFile(PersistenceManager pm, String prefix, byte[] fileData) throws InternalServerException {
try (BufferedReader reader = ServerTools.getAsBufferedReader(fileData)) {
List<StatDataItemStore> items = new LinkedList<>();
JSONArray dataArray = (JSONArray) JSONValue.parse(reader);
for (Object v : dataArray) {
JSONObject entry = (JSONObject) v;
String key = String.format("%s#Q=%s&P=%s", prefix, entry.get("q"), entry.get("p"));
String value = entry.toJSONString();
items.add(new StatDataItemStore(key, value));
}
pm.makePersistentAll(items);
logger.log(Level.INFO, "Stored " + items.size() + " statistical data items for prefix '" + prefix + "'");
} catch (IOException e) {
throw new InternalServerException("Failed to close statistical data file", e);
}
}
}