DataStoreFileUploadTest.java

/* (c) 2017 Open Source Geospatial Foundation - all rights reserved
 * This code is licensed under the GPL 2.0 license, available at the root
 * application directory.
 */
package org.geoserver.rest.catalog;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.geoserver.catalog.Catalog;
import org.geoserver.catalog.DataStoreInfo;
import org.geoserver.catalog.NamespaceInfo;
import org.geoserver.catalog.WorkspaceInfo;
import org.geoserver.catalog.impl.NamespaceInfoImpl;
import org.geoserver.catalog.impl.WorkspaceInfoImpl;
import org.geoserver.data.test.MockData;
import org.geoserver.data.test.SystemTestData;
import org.geoserver.filters.LoggingFilter;
import org.geoserver.platform.GeoServerResourceLoader;
import org.geoserver.platform.resource.Files;
import org.geotools.data.DataUtilities;
import org.h2.tools.DeleteDbFiles;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.mock.web.MockHttpServletResponse;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.servlet.Filter;
import java.io.*;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;

import static org.geoserver.rest.RestBaseController.ROOT_PATH;
import static org.junit.Assert.*;

public class DataStoreFileUploadTest extends CatalogRESTTestSupport {
    @Override
    protected void onSetUp(SystemTestData testData) throws Exception {
        super.onSetUp(testData);

        NamespaceInfo gsmlNamespace = new NamespaceInfoImpl();
        gsmlNamespace.setPrefix("gsml");
        gsmlNamespace.setURI("http://www.cgi-iugs.org/xml/GeoSciML/2");

        WorkspaceInfo gsmlWorkspace = new WorkspaceInfoImpl();
        gsmlWorkspace.setName("gsml");

        getCatalog().add(gsmlNamespace);
        getCatalog().add(gsmlWorkspace);
    }

    @Override
    protected List<Filter> getFilters() {
        LoggingFilter filter = new LoggingFilter();
        filter.setEnabled(true);
        filter.setLogBodies(true);
        return Collections.singletonList(filter);
    }

    @Before
    public void removePdsDataStore() {
        removeStore("gs", "pds");
        removeStore("gs", "store with spaces");
    }

    @After
    public void cleanUpDbFiles() throws Exception {
        DeleteDbFiles.execute("target", "foo", true);
        DeleteDbFiles.execute("target", "pds", true);
        DeleteDbFiles.execute("target", "chinese_poly", true);
    }

    @Test
    public void testPropertyFileUpload() throws Exception {
        /*
        Properties p = new Properties();
        p.put( "_", "name:String,pointProperty:Point");
        p.put( "pds.0", "'zero'|POINT(0 0)");
        p.put( "pds.1", "'one'|POINT(1 1)");
        */
        byte[] bytes = propertyFile();
        //p.store( output, null );

        put( ROOT_PATH+"/workspaces/gs/datastores/pds/file.properties", bytes, "text/plain");
        Document dom = getAsDOM( "wfs?request=getfeature&typename=gs:pds" );
        assertFeatures( dom );
    }

    @Test
    public void testPropertyFileUploadWithWorkspace() throws Exception {
        byte[] bytes = propertyFile();

        put( ROOT_PATH+"/workspaces/sf/datastores/pds/file.properties", bytes, "text/plain");
        Document dom = getAsDOM( "wfs?request=getfeature&typename=sf:pds");
        assertFeatures( dom, "sf" );
    }

    @Test
    public void testPropertyFileUploadZipped() throws Exception {
        byte[] bytes = propertyFile();

        //compress
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        ZipOutputStream zout = new ZipOutputStream( out );
        zout.putNextEntry( new ZipEntry( "pds.properties" ) );
        zout.write( bytes );
        zout.flush();
        zout.close();

        put( ROOT_PATH+"/workspaces/gs/datastores/pds/file.properties", out.toByteArray(), "application/zip");

        Document dom = getAsDOM( "wfs?request=getfeature&typename=gs:pds" );
        assertFeatures( dom );

    }

    byte[] propertyFile() throws IOException {
        ByteArrayOutputStream output = new ByteArrayOutputStream();
        BufferedWriter writer = new BufferedWriter( new OutputStreamWriter( output ) );
        writer.write( "_=name:String,pointProperty:Point\n" );
        writer.write( "ds.0='zero'|POINT(0 0)\n");
        writer.write( "ds.1='one'|POINT(1 1)\n");
        writer.flush();
        return output.toByteArray();
    }

    void assertFeatures( Document dom ) throws Exception {
        assertFeatures( dom, "gs" );
    }

    void assertFeatures( Document dom, String ns ) throws Exception {
        assertEquals( "wfs:FeatureCollection", dom.getDocumentElement().getNodeName() );
        assertEquals( 2, dom.getElementsByTagName( ns + ":pds").getLength() );
    }

    @Test
    public void testShapeFileUpload() throws Exception {
        byte[] bytes = shpZipAsBytes();
        put( ROOT_PATH+"/workspaces/gs/datastores/pds/file.shp", bytes, "application/zip");
        Document dom = getAsDOM( "wfs?request=getfeature&typename=gs:pds" );
        assertFeatures( dom );
    }

    @Test
    public void testShapeFileUploadWithCharset() throws Exception {
        /* Requires that a zipped shapefile (chinese_poly.zip) be in test-data directory */
        byte[] bytes = shpChineseZipAsBytes();
        MockHttpServletResponse response =
                putAsServletResponse(ROOT_PATH+"/workspaces/gs/datastores/chinese_poly/file.shp?charset=UTF-8", bytes, "application/zip");
        assertEquals( 201, response.getStatus() );

        MockHttpServletResponse response2 =
                getAsServletResponse("wfs?request=getfeature&typename=gs:chinese_poly", "GB18030");
        assertTrue(response2.getContentAsString().contains("\u951f\u65a4\u62f7"));
    }

    byte[] shpZipAsBytes() throws IOException {
        return toBytes(getClass().getResourceAsStream( "test-data/pds.zip" ));
    }

    byte[] shpChineseZipAsBytes() throws IOException {
        return toBytes(getClass().getResourceAsStream( "test-data/chinese_poly.zip" ));
    }

    byte[] shpMultiZipAsBytes() throws IOException {
        return toBytes(getClass().getResourceAsStream( "test-data/pdst.zip" ));
    }

    byte[] toBytes(InputStream in) throws IOException {
        ByteArrayOutputStream out = new ByteArrayOutputStream();

        int c;
        while ( ( c = in.read() ) != -1 ) {
            out.write( c );
        }
        return out.toByteArray();
    }

    @Test
    public void testShapeFileUploadExternal() throws Exception {
        Document dom = getAsDOM( "wfs?request=getfeature&typename=gs:pds" );
        assertEquals("ows:ExceptionReport", dom.getDocumentElement().getNodeName());

        File target = new File("target");
        File f = File.createTempFile("rest", "dir", target);
        try {
            f.delete();
            f.mkdir();

            File zip = new File(f, "pds.zip");
            IOUtils.copy(getClass().getResourceAsStream("test-data/pds.zip"), new FileOutputStream(
                    zip));
            org.geoserver.rest.util.IOUtils.inflate(new ZipFile(zip), Files.asResource(f), null);

            MockHttpServletResponse resp = putAsServletResponse(
                    ROOT_PATH+"/workspaces/gs/datastores/pds/external.shp", new File(f, "pds.shp")
                            .toURL().toString(), "text/plain");
            assertEquals(201, resp.getStatus());

            dom = getAsDOM("wfs?request=getfeature&typename=gs:pds");
            assertFeatures(dom);

            // try to download it again after a full reload from disk (GEOS-4616)
            getGeoServer().reload();

            resp = getAsServletResponse(ROOT_PATH+"/workspaces/gs/datastores/pds/file.shp");
            assertEquals( 200, resp.getStatus() );
            assertEquals( "application/zip", resp.getContentType() );

            Set<String> entryNames = new HashSet<>();
            try(ByteArrayInputStream bin = getBinaryInputStream(resp); ZipInputStream zin = new ZipInputStream( bin )) {
                ZipEntry entry;
                while((entry= zin.getNextEntry()) != null) {
                    entryNames.add(entry.getName());
                }
            }
            assertTrue(entryNames.contains("pds.shp"));
            assertTrue(entryNames.contains("pds.shx"));
            assertTrue(entryNames.contains("pds.dbf"));
        } finally {
            FileUtils.deleteQuietly(f);
        }
    }

    @Test
    public void testShapeFileUploadNotExisting() throws Exception {
        File file = new File("./target/notThere.tiff");
        if(file.exists()) {
            assertTrue(file.delete());
        }

        URL url = DataUtilities.fileToURL(file.getCanonicalFile());
        String body = url.toExternalForm();
        MockHttpServletResponse response = putAsServletResponse(ROOT_PATH+"/workspaces/gs/datastores/pds/external.shp",
                body, "text/plain");
        assertEquals(400, response.getStatus());
    }

    @Test
    public void testShapeFileUploadIntoExisting() throws Exception {
        Catalog cat = getCatalog();
        assertNull(cat.getDataStoreByName("gs", "foo_h2"));

        String xml =
                "<dataStore>" +
                        " <name>foo_h2</name>" +
                        " <type>H2</type>" +
                        " <connectionParameters>" +
                        "<namespace>" + MockData.DEFAULT_URI + "</namespace>" +
                        "<database>target/foo</database>" +
                        "<dbtype>h2</dbtype>" +
                        " </connectionParameters>" +
                        "<workspace>gs</workspace>" +
                        "</dataStore>";

        post(ROOT_PATH+"/workspaces/gs/datastores", xml);

        DataStoreInfo ds = cat.getDataStoreByName("gs", "foo_h2");
        assertNotNull(ds);

        assertTrue(cat.getFeatureTypesByDataStore(ds).isEmpty());

        byte[] bytes = shpZipAsBytes();
        put( ROOT_PATH+"/workspaces/gs/datastores/foo_h2/file.shp", bytes, "application/zip");

        assertFalse(cat.getFeatureTypesByDataStore(ds).isEmpty());

        Document dom = getAsDOM( "wfs?request=getfeature&typename=gs:pds" );
        assertFeatures( dom );
    }

    @Test
    public void testShapeFileUploadWithTarget() throws Exception {
        Catalog cat = getCatalog();
        assertNull(cat.getDataStoreByName("gs", "pds"));

        byte[] bytes = shpZipAsBytes();
        put( ROOT_PATH+"/workspaces/gs/datastores/pds/file.shp?target=h2", bytes, "application/zip");

        DataStoreInfo ds = cat.getDataStoreByName("gs", "pds");
        assertNotNull(ds);
        assertFalse(cat.getFeatureTypesByDataStore(ds).isEmpty());

        Document dom = getAsDOM( "wfs?request=getfeature&typename=gs:pds" );
        assertFeatures( dom );
    }

    @Test
    @Ignore
    // fixing https://osgeo-org.atlassian.net/browse/GEOS-6845, re-enable when a proper fix for spaces in
    // name has been made
    public void testShapeFileUploadWithSpaces() throws Exception {
        Catalog cat = getCatalog();
        assertNull(cat.getDataStoreByName("gs", "store with spaces"));

        byte[] bytes = shpZipAsBytes();
        put( ROOT_PATH+"/workspaces/gs/datastores/store%20with%20spaces/file.shp", bytes, "application/zip");

        DataStoreInfo ds = cat.getDataStoreByName("gs", "store with spaces");
        assertNull(ds);
    }

    @Test
    public void testShapefileUploadMultiple() throws Exception {
        Catalog cat = getCatalog();
        assertNull(cat.getDataStoreByName("gs", "pdst"));

        put(ROOT_PATH+"/workspaces/gs/datastores/pdst/file.shp?configure=all", shpMultiZipAsBytes(), "application/zip");

        DataStoreInfo ds = cat.getDataStoreByName("gs", "pdst");
        assertNotNull(ds);

        assertEquals(2, cat.getFeatureTypesByDataStore(ds).size());
    }

    @Test
    public void testGetProperties() throws Exception {
        MockHttpServletResponse resp = getAsServletResponse(ROOT_PATH+"/workspaces/gs/datastores/pds/file.properties");
        assertEquals( 404, resp.getStatus() );

        byte[] bytes = propertyFile();
        put( ROOT_PATH+"/workspaces/gs/datastores/pds/file.properties", bytes, "text/plain");

        resp = getAsServletResponse(ROOT_PATH+"/workspaces/gs/datastores/pds/file.properties");
        assertEquals( 200, resp.getStatus() );
        assertEquals( "application/zip", resp.getContentType() );

        ByteArrayInputStream bin = getBinaryInputStream(resp);
        ZipInputStream zin = new ZipInputStream( bin );

        ZipEntry entry = zin.getNextEntry();
        assertNotNull( entry );
        assertEquals( "pds.properties", entry.getName() );
    }

    @Test
    public void testAppSchemaMappingFileUpload() throws Exception {
        byte[] bytes = appSchemaMappingAsBytes();
        if (bytes == null) {
            // skip test
            LOGGER.warning("app-schema test data not available: skipping test");
            return;
        }

        // copy necessary .properties files from classpath
        loadAppSchemaTestData();

        // upload mapping file (datastore is created implicitly)
        put( ROOT_PATH+"/workspaces/gsml/datastores/mappedPolygons/file.appschema", bytes, "text/xml");
        Document dom = getAsDOM( "wfs?request=getfeature&typename=gsml:MappedFeature" );

        // print(dom);

        assertEquals( "wfs:FeatureCollection", dom.getDocumentElement().getNodeName() );
        NodeList mappedFeatureNodes = dom.getDocumentElement().getElementsByTagNameNS(
                "http://www.cgi-iugs.org/xml/GeoSciML/2", "MappedFeature");
        assertNotNull(mappedFeatureNodes);
        assertEquals(2, mappedFeatureNodes.getLength());

        int namesCount = countNameAttributes(mappedFeatureNodes.item(0));
        assertEquals(2, namesCount);

        // upload alternative mapping file
        bytes = appSchemaAlternativeMappingAsBytes();
        put(ROOT_PATH+"/workspaces/gsml/datastores/mappedPolygons/file.appschema?configure=none",
                bytes, "text/xml");
        dom = getAsDOM( "wfs?request=getfeature&typename=gsml:MappedFeature" );

        // print(dom);

        assertEquals( "wfs:FeatureCollection", dom.getDocumentElement().getNodeName() );
        mappedFeatureNodes = dom.getDocumentElement().getElementsByTagNameNS(
                "http://www.cgi-iugs.org/xml/GeoSciML/2", "MappedFeature");
        assertNotNull(mappedFeatureNodes);
        assertEquals(2, mappedFeatureNodes.getLength());

        namesCount = countNameAttributes(mappedFeatureNodes.item(0));
        // just one name should be found
        assertEquals(1, namesCount);
    }

    private int countNameAttributes(Node mappedFeatureNode) {
        NodeList attrNodes = mappedFeatureNode.getChildNodes();
        int namesCount = 0;
        for (int i=0; i<attrNodes.getLength(); i++) {
            Node attribute = attrNodes.item(i);
            if ("name".equals(attribute.getLocalName())) {
                namesCount++;
            }
        }

        return namesCount;
    }

    private void loadAppSchemaTestData() throws IOException {
        GeoServerResourceLoader loader = new GeoServerResourceLoader(getTestData()
                .getDataDirectoryRoot());
        loader.copyFromClassPath("test-data/mappedPolygons.properties",
                "data/gsml/mappedPolygons.properties");
        loader.copyFromClassPath("test-data/mappedPolygons.oasis.xml",
                "data/gsml/mappedPolygons.oasis.xml");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/CGI_basicTypes.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/CGI_basicTypes.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/CGI_Value.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/CGI_Value.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/earthMaterial.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/earthMaterial.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/fossil.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/fossil.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/geologicStructure.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/geologicStructure.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/geologicUnit.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/geologicUnit.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/geosciml.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/geosciml.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/Gsml.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/Gsml.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/metadata.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/metadata.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/ObsAndMeas.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/ObsAndMeas.xsd");
        loader.copyFromClassPath(
                "test-data/commonSchemas_new/GeoSciML/vocabulary.xsd",
                "data/gsml/commonSchemas_new/GeoSciML/vocabulary.xsd");
    }

    private byte[] appSchemaMappingAsBytes() throws IOException {
        InputStream in = getClass().getResourceAsStream( "/test-data/mappedPolygons.xml" );
        if (in != null) {
            byte[] original = toBytes(in);
            byte[] modified;

            String originalAsString = new String(original, Charset.forName("UTF-8"));
            // modify paths in the original mapping file
            String modifiedAsString = originalAsString.replace("file:./", "file:../")
                    .replace("commonSchemas_new/", "../commonSchemas_new/")
                    .replace("mappedPolygons.oasis", "../mappedPolygons.oasis");

            modified = modifiedAsString.getBytes(Charset.forName("UTF-8"));

            return modified;
        } else {
            return null;
        }
    }

    private byte[] appSchemaAlternativeMappingAsBytes() throws Exception {
        byte[] mapping = appSchemaMappingAsBytes();
        if (mapping != null) {
            Document mappingDom = dom(new ByteArrayInputStream(mapping));

            // remove mapping for MappedFeature/gml:name[2] attribute
            NodeList attrMappingNodes = mappingDom.getDocumentElement()
                    .getElementsByTagName("AttributeMapping");
            for (int i=0; i<attrMappingNodes.getLength(); i++) {
                Node attrMapping = attrMappingNodes.item(i);
                NodeList children = attrMapping.getChildNodes();
                for (int j=0; j<children.getLength(); j++) {
                    if ("MappedFeature/gml:name[2]".equals(children.item(j).getTextContent())) {
                        attrMapping.getParentNode().removeChild(attrMapping);
                        break;
                    }
                }
            }

            ByteArrayOutputStream output = new ByteArrayOutputStream();
            print(mappingDom, output);

            return output.toByteArray();
        } else {
            return null;
        }
    }
}