RecordReader.java

/*
 * Copyright 2010 Outerthought bvba
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.lilyproject.tools.import_.json;

import java.io.IOException;
import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
import org.joda.time.DateTime;
import org.joda.time.LocalDate;
import org.joda.time.format.ISODateTimeFormat;
import org.lilyproject.bytes.api.ByteArray;
import org.lilyproject.repository.api.Blob;
import org.lilyproject.repository.api.FieldType;
import org.lilyproject.repository.api.HierarchyPath;
import org.lilyproject.repository.api.LRepository;
import org.lilyproject.repository.api.Link;
import org.lilyproject.repository.api.MetadataBuilder;
import org.lilyproject.repository.api.QName;
import org.lilyproject.repository.api.Record;
import org.lilyproject.repository.api.RepositoryException;
import org.lilyproject.repository.api.ValueType;
import org.lilyproject.util.json.JsonUtil;

import static org.lilyproject.util.json.JsonUtil.getArray;
import static org.lilyproject.util.json.JsonUtil.getObject;
import static org.lilyproject.util.json.JsonUtil.getString;

public class RecordReader implements EntityReader<Record> {
    public static final RecordReader INSTANCE = new RecordReader();
    private final LinkTransformer defaultLinkTransformer = new DefaultLinkTransformer();

    @Override
    public Record fromJson(JsonNode node, LRepository repository) throws JsonFormatException, RepositoryException,
            InterruptedException {
        return fromJson(node, null, repository);
    }

    @Override
    public Record fromJson(JsonNode nodeNode, Namespaces namespaces, LRepository repository)
            throws JsonFormatException, RepositoryException, InterruptedException {
        return fromJson(nodeNode, namespaces, repository, defaultLinkTransformer);
    }


    @Override
    public Record fromJson(JsonNode nodeNode, Namespaces namespaces, LRepository repository,
            LinkTransformer linkTransformer)
            throws JsonFormatException, RepositoryException, InterruptedException {

        if (!nodeNode.isObject()) {
            throw new JsonFormatException("Expected a json object for record, got: " +
                    nodeNode.getClass().getName());
        }

        ObjectNode node = (ObjectNode)nodeNode;

        namespaces = NamespacesConverter.fromContextJson(node, namespaces);

        return readRootRecord(new ValueHandle(node, "(root object)", null),
                new ReadContext(repository, namespaces, linkTransformer));
    }

    protected Record readRootRecord(ValueHandle handle, ReadContext context)
            throws InterruptedException, RepositoryException, JsonFormatException {
        LRepository repository = context.repository;
        Namespaces namespaces = context.namespaces;
        JsonNode node = handle.node;

        Record record = readCommonRecordAspects(handle, context, true);

        String id = getString(node, "id", null);
        if (id != null) {
            record.setId(repository.getIdGenerator().fromString(id));
        }

        ArrayNode fieldsToDelete = getArray(node, "fieldsToDelete", null);
        if (fieldsToDelete != null) {
            for (int i = 0; i < fieldsToDelete.size(); i++) {
                JsonNode fieldToDelete = fieldsToDelete.get(i);
                if (!fieldToDelete.isTextual()) {
                    throw new JsonFormatException("fieldsToDelete should be an array of strings, encountered: " + fieldToDelete);
                } else {
                    QName qname = QNameConverter.fromJson(fieldToDelete.getTextValue(), namespaces);
                    record.getFieldsToDelete().add(qname);
                }
            }
        }

        ObjectNode attributes = getObject(node, "attributes", null);
        if (attributes != null) {
            Iterator<Map.Entry<String, JsonNode>> it = attributes.getFields();
            while(it.hasNext()) {
                Map.Entry<String, JsonNode> entry = it.next();
                record.getAttributes().put(entry.getKey(), entry.getValue().getTextValue());
            }
        }

        Map<QName, MetadataBuilder> metadataBuilders = null;
        ObjectNode metadata = getObject(node, "metadata", null);
        if (metadata != null) {
            metadataBuilders = new HashMap<QName, MetadataBuilder>();
            Iterator<Map.Entry<String, JsonNode>> it = metadata.getFields();
            while (it.hasNext()) {
                Map.Entry<String, JsonNode> entry = it.next();
                QName qname = QNameConverter.fromJson(entry.getKey(), namespaces);
                MetadataBuilder builder = readMetadata(entry.getValue(), qname);
                metadataBuilders.put(qname, builder);
            }
        }

        ObjectNode metadataToDelete = getObject(node, "metadataToDelete", null);
        if (metadataToDelete != null) {
            if (metadataBuilders == null) {
                metadataBuilders = new HashMap<QName, MetadataBuilder>();
            }

            Iterator<Map.Entry<String, JsonNode>> it = metadataToDelete.getFields();
            while (it.hasNext()) {
                Map.Entry<String, JsonNode> entry = it.next();
                QName qname = QNameConverter.fromJson(entry.getKey(), namespaces);
                MetadataBuilder builder = readMetadataToDelete(entry.getValue(), metadataBuilders.get(qname), qname);
                metadataBuilders.put(qname, builder);
            }
        }

        if (metadataBuilders != null) {
            for (Map.Entry<QName, MetadataBuilder> entry : metadataBuilders.entrySet()) {
                record.setMetadata(entry.getKey(), entry.getValue().build());
            }
        }

        return record;
    }

    protected Record readNestedRecord(ValueHandle handle, ReadContext context)
            throws InterruptedException, RepositoryException, JsonFormatException {
        return readCommonRecordAspects(handle, context, false);
    }

    /**
     * Reads those aspects of a record that are shared between top-level and nested records.
     */
    protected Record readCommonRecordAspects(ValueHandle handle, ReadContext context, boolean topLevelRecord)
            throws JsonFormatException, InterruptedException, RepositoryException {
        LRepository repository = context.repository;
        Namespaces namespaces = context.namespaces;

        Record record = repository.getRecordFactory().newRecord();

        JsonNode typeNode = handle.node.get("type");
        if (typeNode != null) {
            if (typeNode.isObject()) {
                QName qname = QNameConverter.fromJson(JsonUtil.getString(typeNode, "name"), namespaces);
                Long version = JsonUtil.getLong(typeNode, "version", null);
                record.setRecordType(qname, version);
            } else if (typeNode.isTextual()) {
                record.setRecordType(QNameConverter.fromJson(typeNode.getTextValue(), namespaces));
            }
        }

        ObjectNode fields = getObject(handle.node, "fields", null);
        if (fields != null) {
            Iterator<Map.Entry<String, JsonNode>> it = fields.getFields();
            while (it.hasNext()) {
                Map.Entry<String, JsonNode> entry = it.next();

                QName qname = QNameConverter.fromJson(entry.getKey(), namespaces);
                FieldType fieldType = repository.getTypeManager().getFieldTypeByName(qname);
                ValueHandle subHandle = new ValueHandle(fields.get(entry.getKey()), "fields." + entry.getKey(),
                        fieldType.getValueType());
                Object value = readValue(subHandle, context);
                if (value != null) {
                    record.setField(qname, value);
                } else if (value == null && deleteNullFields() && topLevelRecord) {
                    record.delete(qname, true);
                }
            }
        }

        return record;
    }

    /**
     * Should fields in the root record, whose value is null, be added to the fields-to-delete?
     * Subclasses can override this to trigger this behavior (in normal json import, fields
     * will never be null, it is only by subclasses overriding the value-reading methods
     * that values can become null).
     */
    protected boolean deleteNullFields() {
        return false;
    }

    protected List<Object> readList(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        JsonNode node = handle.node;
        if (!node.isArray()) {
            throw new JsonFormatException("List value should be specified as array in " + handle.prop);
        }

        List<Object> value = new ArrayList<Object>();
        for (int i = 0; i < node.size(); i++) {
            ValueHandle subHandle = new ValueHandle(node.get(i), handle.prop + "[" + i + "]",
                    handle.valueType.getNestedValueType());
            Object subValue = readValue(subHandle, context);
            if (subValue != null) {
                value.add(subValue);
            }
        }

        return value;
    }

    protected List<Object> readPath(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {

        JsonNode node = handle.node;

        if (!node.isArray()) {
            throw new JsonFormatException("Path value should be specified as an array in " + handle.prop);
        }

        List<Object> elements = new ArrayList<Object>(node.size());
        for (int i = 0; i < node.size(); i++) {
            ValueHandle subHandle = new ValueHandle(node.get(i), handle.prop,
                    handle.valueType.getNestedValueType());
            Object subValue = readValue(subHandle, context);
            if (subValue != null) {
                elements.add(subValue);
            }
        }

        return new HierarchyPath(elements.toArray(new Object[elements.size()]));
    }

    protected String readString(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isTextual()) {
            throw new JsonFormatException("Expected text value for property '" + handle.prop + "'");
        }
        return handle.node.getTextValue();
    }

    protected Integer readInteger(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (handle.node.isIntegralNumber()) {
            return handle.node.getIntValue();
        } else if (handle.node.isTextual()) {
            try {
                return Integer.parseInt(handle.node.getTextValue());
            } catch (NumberFormatException e) {
                throw new JsonFormatException(String.format("Unparsable int value in property '%s': %s", handle.prop,
                        handle.node.getTextValue()));
            }
        } else {
            throw new JsonFormatException("Expected int value for property '" + handle.prop + "'");
        }
    }

    protected Long readLong(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (handle.node.isIntegralNumber()) {
            return handle.node.getLongValue();
        } else if (handle.node.isTextual()) {
            try {
                return Long.parseLong(handle.node.getTextValue());
            } catch (NumberFormatException e) {
                throw new JsonFormatException(String.format("Unparsable long value in property '%s': %s", handle.prop,
                        handle.node.getTextValue()));
            }
        } else {
            throw new JsonFormatException("Expected long value for property '" + handle.prop + "'");
        }
    }

    protected Double readDouble(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (handle.node.isNumber()) {
            return handle.node.getDoubleValue();
        } else if (handle.node.isTextual()) {
            try {
                return Double.parseDouble(handle.node.getTextValue());
            } catch (NumberFormatException e) {
                throw new JsonFormatException(String.format("Unparsable double value in property '%s': %s", handle.prop,
                        handle.node.getTextValue()));
            }
        } else {
            throw new JsonFormatException("Expected double value for property '" + handle.prop + "'");
        }
    }

    protected BigDecimal readDecimal(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (handle.node.isNumber()) {
            return handle.node.getDecimalValue();
        } else if (handle.node.isTextual()) {
            try {
                return new BigDecimal(handle.node.getTextValue());
            } catch (NumberFormatException e) {
                throw new JsonFormatException(String.format("Unparsable decimal value in property '%s': %s", handle.prop,
                        handle.node.getTextValue()));
            }
        } else {
            throw new JsonFormatException("Expected decimal value for property '" + handle.prop + "'");
        }
    }

    protected URI readUri(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isTextual()) {
            throw new JsonFormatException("Expected URI (string) value for property '" + handle.prop + "'");
        }

        try {
            return new URI(handle.node.getTextValue());
        } catch (URISyntaxException e) {
            throw new JsonFormatException("Invalid URI in property '" + handle.prop + "': "
                    + handle.node.getTextValue());
        }
    }

    protected Boolean readBoolean(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (handle.node.isBoolean()) {
            return handle.node.getBooleanValue();
        } else if (handle.node.isTextual()) {
            String text = handle.node.getTextValue();
            // I think being strict in what to accept is more user friendly, rather than considering everything
            // that is not recognized to be false
            if (text.equalsIgnoreCase("true") || text.equalsIgnoreCase("t")) {
                return Boolean.TRUE;
            } else if (text.equalsIgnoreCase("false") || text.equalsIgnoreCase("f")) {
                return Boolean.FALSE;
            } else {
                throw new JsonFormatException(String.format("Unparsable boolean value in property '%s': %s", handle.prop,
                        handle.node.getTextValue()));
            }
        } else {
            throw new JsonFormatException("Expected boolean value for property '" + handle.prop + "'");
        }
    }

    protected Link readLink(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isTextual()) {
            throw new JsonFormatException("Expected text value for property '" + handle.prop + "'");
        }

        return context.linkTransformer.transform(handle.node.getTextValue(), context.repository);
    }

    protected LocalDate readDate(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isTextual()) {
            throw new JsonFormatException("Expected text value for property '" + handle.prop + "'");
        }

        return new LocalDate(handle.node.getTextValue());
    }

    protected DateTime readDateTime(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isTextual()) {
            throw new JsonFormatException("Expected text value for property '" + handle.prop + "'");
        }

        return new DateTime(handle.node.getTextValue());
    }

    protected Blob readBlob(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isObject()) {
            throw new JsonFormatException("Expected object value for property '" + handle.prop + "'");
        }

        ObjectNode blobNode = (ObjectNode)handle.node;
        return BlobConverter.fromJson(blobNode);
    }

    protected ByteArray readByteArray(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {
        if (!handle.node.isTextual()) {
            throw new JsonFormatException("Expected base64 encoded value for property '" + handle.prop + "'");
        }
        try {
            return new ByteArray(handle.node.getBinaryValue());
        } catch (IOException e) {
            throw new JsonFormatException("Could not read base64 value for property '" + handle.prop + "'", e);
        }
    }

    /**
     * Reads/parses the JSON serialization of a value following a Lily {@link ValueType}. While typically this
     * will be the value of a Lily field in a record, such values might also occur in other places (e.g. a
     * scan filter or a mutation condition) and this method can also be called from there.
     */
    public Object readValue(ValueHandle handle, ReadContext context)
            throws JsonFormatException, RepositoryException, InterruptedException {

        String name = handle.valueType.getBaseName();

        if (name.equals("LIST")) {
            return readList(handle, context);
        } else if (name.equals("PATH")) {
            return readPath(handle, context);
        } else if (name.equals("STRING")) {
            return readString(handle, context);
        } else if (name.equals("INTEGER")) {
            return readInteger(handle, context);
        } else if (name.equals("LONG")) {
            return readLong(handle, context);
        } else if (name.equals("DOUBLE")) {
            return readDouble(handle, context);
        } else if (name.equals("DECIMAL")) {
            return readDecimal(handle, context);
        } else if (name.equals("URI")) {
            return readUri(handle, context);
        } else if (name.equals("BOOLEAN")) {
            return readBoolean(handle, context);
        } else if (name.equals("LINK")) {
            return readLink(handle, context);
        } else if (name.equals("DATE")) {
            return readDate(handle, context);
        } else if (name.equals("DATETIME")) {
            return readDateTime(handle, context);
        } else if (name.equals("BLOB")) {
            return readBlob(handle, context);
        } else if (name.equals("BYTEARRAY")) {
            return readByteArray(handle, context);
        } else if (name.equals("RECORD")) {
            return readNestedRecord(handle, context);
        } else {
            throw new JsonFormatException("Value type not supported: " + name);
        }
    }

    /**
     * Information on a value to parse: the JSON node containing the value, the property in which it occurs,
     * and its Lily ValueType.
     */
    public static class ValueHandle {
        /** Node representing the value to parse. */
        JsonNode node;
        /** JSON property name in which the value occurs, used in error messages. */
        String prop;
        /** Lily value type of the value to parse. */
        ValueType valueType;

        public ValueHandle(JsonNode node, String prop, ValueType valueType) {
            this.node = node;
            this.prop = prop;
            this.valueType = valueType;
        }
    }

    /**
     * Global context accessible while parsing values.
     */
    public static class ReadContext {
        LRepository repository;
        LinkTransformer linkTransformer;
        Namespaces namespaces;
        
        public ReadContext(LRepository repository, Namespaces namespaces, LinkTransformer linkTransformer) {
            this.repository = repository;
            this.namespaces = namespaces;
            this.linkTransformer = linkTransformer;
        }
    }

    private MetadataBuilder readMetadata(JsonNode metadata, QName recordField) throws JsonFormatException {
        if (!metadata.isObject()) {
            throw new JsonFormatException("The value for the metadata should be an object, field: " + recordField);
        }

        ObjectNode object = (ObjectNode)metadata;
        MetadataBuilder builder = new MetadataBuilder();

        Iterator<Map.Entry<String, JsonNode>> it = object.getFields();
        while (it.hasNext()) {
            Map.Entry<String, JsonNode> entry = it.next();
            String name = entry.getKey();
            JsonNode value = entry.getValue();

            if (value.isTextual()) {
                builder.value(name, value.getTextValue());
            } else if (value.isInt()) {
                builder.value(name, value.getIntValue());
            } else if (value.isLong()) {
                builder.value(name, value.getLongValue());
            } else if (value.isBoolean()) {
                builder.value(name, value.getBooleanValue());
            } else if (value.isFloatingPointNumber()) {
                // In the JSON format, for simplicity, we don't make distinction between float & double, so you
                // can't control which of the two is created.
                builder.value(name, value.getDoubleValue());
            } else if (value.isObject()) {
                String type = JsonUtil.getString(value, "type", null);
                if (type == null) {
                    throw new JsonFormatException("Missing required 'type' property on object in metadata field '"
                            + name + "' of record field " + recordField);
                }

                if (type.equals("binary")) {
                    JsonNode binaryValue = value.get("value");
                    if (!binaryValue.isTextual()) {
                        throw new JsonFormatException("Invalid binary value for metadata field '"
                                + name + "' of record field " + recordField);
                    }

                    try {
                        builder.value(name, new ByteArray(binaryValue.getBinaryValue()));
                    } catch (IOException e) {
                        throw new JsonFormatException("Invalid binary value for metadata field '"
                                + name + "' of record field " + recordField);
                    }
                } else if (type.equals("datetime")) {
                    JsonNode datetimeValue = value.get("value");
                    if (!datetimeValue.isTextual()) {
                        throw new JsonFormatException("Invalid datetime value for metadata field '"
                                + name + "' of record field " + recordField);
                    }

                    try {
                        builder.value(name, ISODateTimeFormat.dateTime().parseDateTime(datetimeValue.getTextValue()));
                    } catch (Exception e) {
                        throw new JsonFormatException("Invalid datetime value for metadata field '"
                                + name + "' of record field " + recordField);
                    }
                } else {
                    throw new JsonFormatException("Unsupported type value '" + type + "' for metadata field '"
                            + name + "' of record field " + recordField);
                }
            } else {
                throw new JsonFormatException("Unsupported type of value for metadata field '" + name
                        + "' of record field " + recordField);
            }
        }

        return builder;
    }

    private MetadataBuilder readMetadataToDelete(JsonNode metadataToDelete, MetadataBuilder builder,
            QName recordField) throws JsonFormatException {
        if (!metadataToDelete.isArray()) {
            throw new JsonFormatException("The value for the metadataToDelete should be an array, field: " + recordField);
        }

        ArrayNode array = (ArrayNode)metadataToDelete;
        if (builder == null) {
            builder = new MetadataBuilder();
        }

        for (int i = 0; i < array.size(); i++) {
            JsonNode entry = array.get(i);
            if (!entry.isTextual()) {
                throw new JsonFormatException("Non-string found in the metadataToDelete array of field: " + recordField);
            }
            builder.delete(entry.getTextValue());
        }

        return builder;
    }

}