MaintainResource.java

/**
 * This file is part of d:swarm graph extension.
 *
 * d:swarm graph extension is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * d:swarm graph extension is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with d:swarm graph extension.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.dswarm.graph.resources;

import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;

import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;

import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Charsets;
import com.google.common.io.Resources;
import org.mapdb.Atomic;
import org.mapdb.DB;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.Result;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.index.Index;
import org.neo4j.graphdb.schema.IndexDefinition;
import org.neo4j.graphdb.schema.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.dswarm.common.DMPStatics;
import org.dswarm.common.types.Tuple;
import org.dswarm.graph.BasicNeo4jProcessor;
import org.dswarm.graph.DMPGraphException;
import org.dswarm.graph.DataModelNeo4jProcessor;
import org.dswarm.graph.GraphIndexStatics;
import org.dswarm.graph.GraphProcessingStatics;
import org.dswarm.graph.deprecate.DataModelNeo4jDeprecator;
import org.dswarm.graph.deprecate.RecordsNeo4jDeprecator;
import org.dswarm.graph.deprecate.RelationshipDeprecator;
import org.dswarm.graph.index.MapDBUtils;
import org.dswarm.graph.index.NamespaceIndex;
import org.dswarm.graph.index.SchemaIndexUtils;
import org.dswarm.graph.tx.Neo4jTransactionHandler;
import org.dswarm.graph.tx.TransactionHandler;
import org.dswarm.graph.utils.GraphDatabaseUtils;
import org.dswarm.graph.utils.NamespaceUtils;

/**
 * @author tgaengler
 */
@Path("/maintain")
public class MaintainResource extends GraphResource {

	private static final Logger LOG = LoggerFactory.getLogger(MaintainResource.class);

	private static final String PERSISTENT_GRAPH_DATABASE_IDENTIFIER = "persistent graph database";

	private static final long chunkSize = 50000;

	// TODO: maybe divide this into 2 queries and without OPTIONAL
	private static final String DELETE_CYPHER = "MATCH (a) WITH a LIMIT %d OPTIONAL MATCH (a)-[r]-() DELETE a,r RETURN COUNT(*) AS entity_count";

	private static final String DEPRECATE_DATA_MODEL_TYPE = "deprecate data model";

	private static final String DEPRECATE_RECORDS_TYPE = "deprecate records";

	public MaintainResource() {

	}

	@GET
	@Path("/ping")
	public String ping() {

		MaintainResource.LOG.debug("ping was called");

		return "pong";
	}

	@POST
	@Path("/deprecate/datamodel")
	@Consumes(MediaType.APPLICATION_JSON)
	@Produces(MediaType.APPLICATION_JSON)
	public Response deprecateDataModel(final String jsonObjectString, @Context final GraphDatabaseService database) throws DMPGraphException {

		MaintainResource.LOG.info("try to deprecate data model in graph db");

		final ObjectNode requestJSON = deserializeJSON(jsonObjectString, DEPRECATE_DATA_MODEL_TYPE);

		final String dataModelUri = requestJSON.get(DMPStatics.DATA_MODEL_URI_IDENTIFIER).asText();

		final TransactionHandler tx = new Neo4jTransactionHandler(database);
		final NamespaceIndex namespaceIndex = new NamespaceIndex(database, tx);

		final String prefixedDataModelUri = namespaceIndex.createPrefixedURI(dataModelUri);

		MaintainResource.LOG.info("try to deprecate statements in data model '{}' ('{}') in graph db", dataModelUri, prefixedDataModelUri);

		final BasicNeo4jProcessor processor = new DataModelNeo4jProcessor(database, tx, namespaceIndex, prefixedDataModelUri);

		final RelationshipDeprecator dataModelDeprecator = new DataModelNeo4jDeprecator(processor, true, prefixedDataModelUri);

		dataModelDeprecator.work();

		final int relationshipsDeprecated = dataModelDeprecator.getRelationshipsDeprecated();

		if (relationshipsDeprecated > 0) {

			// update data model version only when some statements are deprecated the DB
			dataModelDeprecator.getVersionHandler().updateLatestVersion();
		}

		dataModelDeprecator.closeTransaction();

		LOG.info("deprecated '{}' relationships in data model '{}' ('{}') in graph db", relationshipsDeprecated, dataModelUri, prefixedDataModelUri);

		final ObjectNode resultJSON = simpleObjectMapper.createObjectNode();
		resultJSON.put("deprecated", relationshipsDeprecated);
		final String result = serializeJSON(resultJSON, DEPRECATE_DATA_MODEL_TYPE);

		return Response.ok(result, MediaType.APPLICATION_JSON_TYPE).build();
	}

	@POST
	@Path("/deprecate/records")
	@Consumes(MediaType.APPLICATION_JSON)
	@Produces(MediaType.APPLICATION_JSON)
	public Response deprecateRecords(final String jsonObjectString, @Context final GraphDatabaseService database) throws DMPGraphException {

		MaintainResource.LOG.info("try to deprecate records in a data model in graph db");

		final ObjectNode requestJSON = deserializeJSON(jsonObjectString, DEPRECATE_RECORDS_TYPE);

		final String dataModelUri = requestJSON.get(DMPStatics.DATA_MODEL_URI_IDENTIFIER).asText();

		final Collection<String> recordURIs = getRecordURIs(requestJSON);

		final TransactionHandler tx = new Neo4jTransactionHandler(database);
		final NamespaceIndex namespaceIndex = new NamespaceIndex(database, tx);

		final String prefixedDataModelUri = namespaceIndex.createPrefixedURI(dataModelUri);

		final Collection<String> prefixedRecordURIs = prefixRecordURIs(recordURIs, namespaceIndex);

		MaintainResource.LOG.info("try to deprecate '{}' records in data model '{}' ('{}') in graph db", prefixedRecordURIs.size(), dataModelUri,
				prefixedDataModelUri);

		final BasicNeo4jProcessor processor = new DataModelNeo4jProcessor(database, tx, namespaceIndex, prefixedDataModelUri);

		final RelationshipDeprecator recordsDeprecator = new RecordsNeo4jDeprecator(processor, true, prefixedDataModelUri, prefixedRecordURIs);

		recordsDeprecator.work();

		final int relationshipsDeprecated = recordsDeprecator.getRelationshipsDeprecated();

		if (relationshipsDeprecated > 0) {

			// update data model version only when some statements are deprecated the DB
			recordsDeprecator.getVersionHandler().updateLatestVersion();
		}

		recordsDeprecator.closeTransaction();

		LOG.info("deprecated '{}' records with '{}' relationships in data model '{}' ('{}') in graph db", prefixedRecordURIs.size(),
				relationshipsDeprecated, dataModelUri, prefixedDataModelUri);

		final ObjectNode resultJSON = simpleObjectMapper.createObjectNode();
		resultJSON.put("deprecated", relationshipsDeprecated);
		final String result = serializeJSON(resultJSON, DEPRECATE_DATA_MODEL_TYPE);

		return Response.ok(result, MediaType.APPLICATION_JSON_TYPE).build();

	}

	/**
	 * note utilise this endpoint with care, because it cleans your complete db!
	 *
	 * @param database the graph database
	 */
	@DELETE
	@Path("/delete")
	@Produces("application/json")
	public Response cleanGraph(@Context final GraphDatabaseService database) throws IOException, DMPGraphException {

		MaintainResource.LOG.debug("start cleaning up the db");

		final long deleted = deleteSomeStatements(database);

		MaintainResource.LOG.debug("finished delete-all-entities TXs");

		MaintainResource.LOG.debug("start legacy indices clean-up");

		// TODO: maybe separate index clean-up + observe index clean-up
		// => maybe we also need to do a label + relationship types clean-up ... => this is not supported right now ...

		deleteSomeLegacyIndices(database);

		MaintainResource.LOG.debug("finished legacy indices clean-up");

		MaintainResource.LOG.debug("start schema indices clean-up");

		deleteSomeSchemaIndices(database);

		MaintainResource.LOG.debug("finished schema indices clean-up");

		MaintainResource.LOG.debug("finished cleaning up the db");

		final StringWriter out = new StringWriter();
		final JsonGenerator generator = simpleObjectMapper.getFactory().createGenerator(out);

		generator.writeStartObject();
		generator.writeNumberField("deleted", deleted);
		generator.writeEndObject();
		generator.flush();
		generator.close();

		final String result = out.toString();

		out.flush();
		out.close();

		return Response.ok(result, MediaType.APPLICATION_JSON_TYPE).build();
	}

	@POST
	@Path("/schemaindices")
	public Response createSchemaIndices(@Context final GraphDatabaseService database) throws DMPGraphException {

		SchemaIndexUtils.createSchemaIndices(database, PERSISTENT_GRAPH_DATABASE_IDENTIFIER);

		initPrefixes(database);

		return Response.ok().build();
	}

	@POST
	@Path("/initprefixcounter")
	public Response initPrefixCounter(@Context final GraphDatabaseService database) throws DMPGraphException {

		initPrefixCounterInternal(database);

		return Response.ok().build();
	}

	private long deleteSomeStatements(final GraphDatabaseService database) throws DMPGraphException {

		final String deleteQuery = String.format(DELETE_CYPHER, MaintainResource.chunkSize);

		long deleted = 0;

		int i = 0;

		while (true) {

			i++;

			try (final Transaction tx = database.beginTx()) {

				MaintainResource.LOG
						.debug("try to delete up to {} nodes and their relationships for the {}. time", MaintainResource.chunkSize, i);

				final Result result = database.execute(deleteQuery);

				if (result == null) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. result is empty");

					tx.success();
					tx.close();

					break;
				}

				if (!result.hasNext()) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. result iterator is empty");

					result.close();
					tx.success();
					tx.close();

					break;
				}

				final Map<String, Object> row = result.next();

				if (row == null || row.isEmpty()) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. row map is empty");

					result.close();
					tx.success();
					tx.close();

					break;
				}

				final Entry<String, Object> entry = row.entrySet().iterator().next();

				if (entry == null) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. entry is not available");

					result.close();
					tx.success();
					tx.close();

					break;
				}

				final Object value = entry.getValue();

				if (value == null) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. value is not available");

					result.close();
					tx.success();
					tx.close();

					break;
				}

				if (!entry.getKey().equals("entity_count")) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. entity count is not available");

					result.close();
					tx.success();
					tx.close();

					break;
				}

				final Long count = (Long) value;

				deleted += count;

				MaintainResource.LOG.debug("deleted {} entities", count);

				if (count < chunkSize) {

					MaintainResource.LOG.debug("there are no more results for removal available, i.e. current result is smaller than chunk size");

					result.close();
					tx.success();

					break;
				}

				result.close();
				tx.success();
				tx.close();
			} catch (final Exception e) {

				final String message = "couldn't finish delete-all-entities TX successfully";

				MaintainResource.LOG.error(message, e);

				throw new DMPGraphException(message);
			}
		}

		return deleted;
	}

	private void deleteSomeLegacyIndices(final GraphDatabaseService database) throws DMPGraphException {

		MaintainResource.LOG.debug("start delete legacy indices TX");

		try (final Transaction itx = database.beginTx()) {

			final Index<Node> resources = database.index().forNodes(GraphIndexStatics.RESOURCES_INDEX_NAME);
			final Index<Node> values = database.index().forNodes(GraphIndexStatics.VALUES_INDEX_NAME);
			final Index<Node> resourcesWDataModel = database.index().forNodes(GraphIndexStatics.RESOURCES_W_DATA_MODEL_INDEX_NAME);
			final Index<Node> resourceTypes = database.index().forNodes(GraphIndexStatics.RESOURCE_TYPES_INDEX_NAME);
			final Tuple<Set<Long>, DB> statementHashesMapDBIndexTuple = getOrCreateLongIndex(GraphIndexStatics.STATEMENT_HASHES_INDEX_NAME, database);
			final Index<Relationship> statementUUIDs = database.index().forRelationships(GraphIndexStatics.STATEMENT_UUIDS_INDEX_NAME);

			if (resources != null) {

				MaintainResource.LOG.debug("delete {} legacy index", GraphIndexStatics.RESOURCES_INDEX_NAME);

				resources.delete();
			}

			if (resourcesWDataModel != null) {

				MaintainResource.LOG.debug("delete {} legacy index", GraphIndexStatics.RESOURCES_W_DATA_MODEL_INDEX_NAME);

				resourcesWDataModel.delete();
			}

			if (resourceTypes != null) {

				MaintainResource.LOG.debug("delete {} legacy index", GraphIndexStatics.RESOURCE_TYPES_INDEX_NAME);

				resourceTypes.delete();
			}

			final DB mapDB = statementHashesMapDBIndexTuple.v2();

			if (mapDB.exists(GraphIndexStatics.STATEMENT_HASHES_INDEX_NAME)) {

				MaintainResource.LOG.debug("delete {} mapdb index", GraphIndexStatics.STATEMENT_HASHES_INDEX_NAME);

				mapDB.delete(GraphIndexStatics.STATEMENT_HASHES_INDEX_NAME);
				mapDB.commit();
				mapDB.close();
			}

			if (statementUUIDs != null) {

				MaintainResource.LOG.debug("delete {} legacy index", GraphIndexStatics.STATEMENT_UUIDS_INDEX_NAME);

				statementUUIDs.delete();
			}

			if (values != null) {

				MaintainResource.LOG.debug("delete {} legacy index", GraphIndexStatics.VALUES_INDEX_NAME);

				values.delete();
			}

			itx.success();
			itx.close();
		} catch (final Exception e) {

			final String message = "couldn't finish delete legacy indices TX successfully";

			MaintainResource.LOG.error(message, e);

			throw new DMPGraphException(message);
		}

		MaintainResource.LOG.debug("finished delete legacy indices TX");
	}

	private void deleteSomeSchemaIndices(final GraphDatabaseService database) throws DMPGraphException {

		MaintainResource.LOG.debug("start delete schema indices TX");

		try (final Transaction itx = database.beginTx()) {

			final Schema schema = database.schema();

			if (schema == null) {

				MaintainResource.LOG.debug("no schema available");

				itx.success();
				itx.close();

				return;
			}

			final Iterable<IndexDefinition> indexDefinitions = schema.getIndexes();

			if (indexDefinitions == null) {

				MaintainResource.LOG.debug("no schema indices available");

				itx.success();
				itx.close();

				return;
			}

			for (final IndexDefinition indexDefinition : indexDefinitions) {

				MaintainResource.LOG.debug("drop '{}' : '{}' schema index", indexDefinition.getLabel().name(),
						indexDefinition.getPropertyKeys().iterator().next());

				indexDefinition.drop();
			}

			itx.success();
			itx.close();
		} catch (final Exception e) {

			final String message = "couldn't finish delete schema indices TX successfully";

			MaintainResource.LOG.error(message, e);

			throw new DMPGraphException(message);
		}

		MaintainResource.LOG.debug("finished delete schema indices TX");
	}

	private Tuple<Set<Long>, DB> getOrCreateLongIndex(final String name, final GraphDatabaseService database) throws IOException {

		final String storeDir = GraphDatabaseUtils.determineMapDBIndexStoreDir(database);

		// storeDir + File.separator + MapDBUtils.INDEX_DIR + name
		return MapDBUtils.createOrGetPersistentLongIndexTreeSetGlobalTransactional(storeDir + File.separator + name, name);
	}

	private void initPrefixes(final GraphDatabaseService database) throws DMPGraphException {

		MaintainResource.LOG.debug("start initialising namespaces index");

		final TransactionHandler tx = new Neo4jTransactionHandler(database);
		final NamespaceIndex namespaceIndex = new NamespaceIndex(database, tx);

		tx.ensureRunningTx();

		try {

			final URL prefixesFileURL = Resources.getResource("prefixes.json");
			final String prefixesJSONString = Resources.toString(prefixesFileURL, Charsets.UTF_8);
			final Map<String, String> prefixesNamespacesMap = simpleObjectMapper
					.readValue(prefixesJSONString, new TypeReference<HashMap<String, String>>() {

					});

			for (final Map.Entry<String, String> entry : prefixesNamespacesMap.entrySet()) {

				final String prefix = entry.getKey();
				final String namespace = entry.getValue();

				final Optional<Node> optionalPrefix = NamespaceUtils.getPrefix(namespace, database);

				if (!optionalPrefix.isPresent()) {

					namespaceIndex.addPrefix(namespace, prefix);
				} else {

					final String prefixFromDB = (String) optionalPrefix.get().getProperty(GraphProcessingStatics.PREFIX_PROPERTY);

					MaintainResource.LOG
							.debug("prefix '{}' is already available for namespace '{}', i.e., no further entry with prefix '{}' will be created",
									prefixFromDB, namespace, prefix);
				}
			}

			tx.succeedTx();
		} catch (final Exception e) {

			tx.failTx();

			final String message = "couldn't initialize prefixes successfully";

			LOG.error(message);

			throw new DMPGraphException(message, e);
		}

		MaintainResource.LOG.debug("finished initialising namespaces index");
	}

	private Collection<String> getRecordURIs(final ObjectNode json) {

		final JsonNode recordsNode = json.get(DMPStatics.RECORDS_IDENTIFIER);

		final ArrayList<String> recordURIs = new ArrayList<>();

		for (final JsonNode recordNode : recordsNode) {

			final String recordURI = recordNode.asText();

			recordURIs.add(recordURI);
		}

		return recordURIs;
	}

	private Collection<String> prefixRecordURIs(final Collection<String> recordURIs, final NamespaceIndex namespaceIndex) throws DMPGraphException {

		final ArrayList<String> prefixedRecordURIs = new ArrayList<>();

		for (final String recordURI : recordURIs) {

			final String prefixedRecordURI = namespaceIndex.createPrefixedURI(recordURI);

			prefixedRecordURIs.add(prefixedRecordURI);
		}

		return prefixedRecordURIs;
	}

	private void initPrefixCounterInternal(final GraphDatabaseService database) throws DMPGraphException {

		try (final Transaction tx = database.beginTx()) {

			final ResourceIterator<Node> nodes = database.findNodes(GraphProcessingStatics.PREFIX_LABEL);

			if (nodes == null) {

				LOG.debug("no prefix nodes available - prefix does not need to initialised with a specific value");

				tx.success();
				tx.close();

				return;
			}

			long biggestPrefixNumber = 0;

			while (nodes.hasNext()) {

				final Node prefixNode = nodes.next();

				final Object prefixObject = prefixNode.getProperty(GraphProcessingStatics.PREFIX_PROPERTY, null);

				if (prefixObject == null) {

					LOG.debug("could not find prefix at prefix node '{}'", prefixNode.getId());

					continue;
				}

				final String prefix = (String) prefixObject;

				if (!prefix.startsWith(NamespaceUtils.NAMESPACE_PREFIX_BASE)) {

					// namespace is no candidate for prefix counter number

					continue;
				}

				if (!(prefix.length() > 2)) {

					// namespace is no candidate for prefix counter number

					continue;
				}

				final String prefixNumberString = prefix.substring(2, prefix.length());

				try {

					final Long prefixNumber = Long.valueOf(prefixNumberString);

					if (prefixNumber > biggestPrefixNumber) {

						biggestPrefixNumber = prefixNumber;
					}
				} catch (final NumberFormatException e) {

					LOG.debug("could not convert prefix number string '{}' to a number", prefixNumberString);
				}
			}

			nodes.close();

			// create persistent prefix counter with biggest given prefix number + 1
			biggestPrefixNumber = biggestPrefixNumber + 1;

			final String storeDir = GraphDatabaseUtils.determineMapDBIndexStoreDir(database);

			final Tuple<Atomic.Long, DB> prefixCounterTuple = MapDBUtils
					.createOrGetPersistentLongIndexGlobalTransactional(storeDir + File.separator + GraphIndexStatics.PREFIX_COUNTER_INDEX_NAME,
							GraphIndexStatics.PREFIX_COUNTER_INDEX_NAME, biggestPrefixNumber);
			final DB prefixCounterDB = prefixCounterTuple.v2();

			prefixCounterDB.commit();
			prefixCounterDB.close();

			LOG.info("initialized the prefix counter index with '{}'", biggestPrefixNumber);

			tx.success();
			tx.close();
		} catch (final Exception e) {

			final String message = "couldn't finish init prefix counter TX successfully";

			MaintainResource.LOG.error(message, e);

			throw new DMPGraphException(message);
		}
	}
}