/******************************************************************************* * Copyright (c) 2014 Open Door Logistics (www.opendoorlogistics.com) * All rights reserved. This program and the accompanying materials * are made available under the terms of the GNU Lesser Public License 3.0 * which accompanies this distribution, and is available at * http://www.gnu.org/licenses/lgpl.html * ******************************************************************************/ package com.opendoorlogistics.core.geometry.rog.builder; import java.awt.geom.Point2D; import java.awt.geom.Rectangle2D; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.commons.io.FilenameUtils; import org.geotools.geometry.jts.JTS; import com.opendoorlogistics.api.ODLApi; import com.opendoorlogistics.api.components.ProcessingApi; import com.opendoorlogistics.api.geometry.LatLong; import com.opendoorlogistics.api.geometry.LatLongToScreen; import com.opendoorlogistics.api.geometry.ODLGeom; import com.opendoorlogistics.api.tables.ODLColumnType; import com.opendoorlogistics.api.tables.ODLDatastoreAlterable; import com.opendoorlogistics.api.tables.ODLTableAlterable; import com.opendoorlogistics.api.tables.ODLTableReadOnly; import com.opendoorlogistics.codefromweb.jxmapviewer2.fork.swingx.OSMTileFactoryInfo; import com.opendoorlogistics.codefromweb.jxmapviewer2.fork.swingx.mapviewer.GeoPosition; import com.opendoorlogistics.codefromweb.jxmapviewer2.fork.swingx.mapviewer.TileFactoryInfo; import com.opendoorlogistics.codefromweb.jxmapviewer2.fork.swingx.mapviewer.util.GeoUtil; import com.opendoorlogistics.core.geometry.ImportShapefile; import com.opendoorlogistics.core.geometry.ODLGeomImpl; import com.opendoorlogistics.core.geometry.Spatial; import com.opendoorlogistics.core.geometry.rog.ODLRenderOptimisedGeom; import com.opendoorlogistics.core.geometry.rog.QuadLoader; import com.opendoorlogistics.core.geometry.rog.RogReaderUtils; import com.opendoorlogistics.core.gis.map.transforms.LatLongToScreenImpl; import com.opendoorlogistics.core.gis.map.transforms.TransformGeomToWorldBitmap; import com.opendoorlogistics.core.tables.memory.ODLDatastoreImpl; import com.opendoorlogistics.core.tables.utils.TableUtils; import com.opendoorlogistics.core.utils.LargeList; import com.vividsolutions.jts.geom.Envelope; import com.vividsolutions.jts.geom.Geometry; import com.vividsolutions.jts.geom.Point; import com.vividsolutions.jts.io.WKBWriter; import com.vividsolutions.jts.simplify.TopologyPreservingSimplifier; public class ROGBuilder { private final int nbThreads; private final File shapefile; private final File outfile; private final File tmpFile; private final double pixelTol; private final boolean isNOLPL; private final TileFactoryInfo tileFactoryInfo = new OSMTileFactoryInfo(); private ProcessingApi processingApi = new ProcessingApi() { @Override public ODLApi getApi() { return null; } @Override public boolean isFinishNow() { return false; } @Override public boolean isCancelled() { return false; } @Override public void postStatusMessage(String s) { System.out.println(s); } @Override public void logWarning(String warning) { System.err.println(warning); } }; // private final LZ4Compressor compressor; public ROGBuilder(File shapefile,boolean isNOLPL, double pixelTol, int nbThreads, ProcessingApi processingApi) { this(shapefile, new File(FilenameUtils.removeExtension(shapefile.getPath()) + "." + RogReaderUtils.RENDER_GEOMETRY_FILE_EXT), isNOLPL, pixelTol, nbThreads,processingApi); } private ROGBuilder(File shapefile, File outfile,boolean isNOLPL, double pixelTol, int nbThreads, ProcessingApi processingApi) { this.shapefile = shapefile; this.outfile = outfile; this.pixelTol = pixelTol; this.tmpFile = new File(outfile.getAbsolutePath() + ".tmp"); this.nbThreads = nbThreads; this.isNOLPL =isNOLPL; // this.compressor = LZ4Factory.unsafeInstance().fastCompressor(); Spatial.initSpatial(); if(processingApi!=null){ this.processingApi = processingApi; } } private class RowAllocator { private final int nRows; private volatile int next = 0; public RowAllocator(int nRows) { super(); this.nRows = nRows; this.next = next; } /* * Allocate a row or return -1 if non available */ synchronized int allocateRow() { int ret = -1; if (next < nRows) { ret = next; next++; } return ret; } } private synchronized void postStatusMessage(String s){ processingApi.postStatusMessage(s); } private class RowProcessor implements Callable<Void> { final ODLTableReadOnly table; final int geomCol; final LargeList<ShapeIndex> indices; final int zoom; TransformGeomToWorldBitmap mathTransform; final RowAllocator allocator; final List<PendingWrite> resultsList; final WKBWriter geomWriter = new WKBWriter(); final String baseMessage; RowProcessor(ODLTableReadOnly table, int geomCol, LargeList<ShapeIndex> indices, int zoom, TransformGeomToWorldBitmap mathTransform, RowAllocator allocator,String baseMessage, List<PendingWrite> resultsList) { this.table = table; this.geomCol = geomCol; this.indices = indices; this.zoom = zoom; this.mathTransform = mathTransform; this.allocator = allocator; this.resultsList = resultsList; this.baseMessage = baseMessage; } @Override public Void call() throws Exception { while (true) { int row = allocator.allocateRow(); if (row == -1) { return null; } PendingWrite pending = processRow(row); resultsList.set(row, pending); if (row % 1000 == 0 && row>0) { postStatusMessage(baseMessage + " - processed " + row + " geometries"); } } } private PendingWrite processRow(int row) { ODLGeomImpl geom = (ODLGeomImpl) table.getValueAt(row, geomCol); if (geom == null) { return null; } Geometry wgs = geom.getJTSGeometry(); try { Geometry transformed = JTS.transform(wgs, mathTransform); Envelope bb = transformed.getEnvelopeInternal(); byte[] bytes = null; int nbPoints = 0; if (bb.getWidth() >= pixelTol || bb.getHeight() >= pixelTol) { // simplify transformed = TopologyPreservingSimplifier.simplify(transformed, pixelTol); // geometry can become empty is its too small .. nbPoints = transformed.getNumPoints(); if (nbPoints > 0) { bytes = geomWriter.write(transformed); } } ShapeIndex index = indices.get(row); if (index.rowNb != row) { throw new RuntimeException(); } if (bytes != null) { boolean writeBytes = true; int lastDefinedLevel = index.findLastDefinedLevel(tileFactoryInfo.getMinimumZoomLevel(), zoom); if (lastDefinedLevel != -1) { int lastNbPoints = index.nbPoints[lastDefinedLevel]; int target = (int) Math.round(lastNbPoints * ROGConstants.MINIMUM_REDUCTION_FRACTION_FOR_NEXT_LEVEL); if (nbPoints >= target) { // Use last defined level geometry as current level is not much more simplified index.blockNb[zoom] = ODLRenderOptimisedGeom.USE_LAST_LEVEL; writeBytes = false; } } if (writeBytes) { index.nbPoints[zoom] = nbPoints; return new PendingWrite(index, bytes); } } else { index.blockNb[zoom] = ODLRenderOptimisedGeom.SUBPIXEL; } } catch (Exception e) { throw new RuntimeException(e); } return null; } } @SuppressWarnings("resource") public void build() { try { // Load the shapefile ODLDatastoreAlterable<ODLTableAlterable> ds = ODLDatastoreImpl.alterableFactory.create(); ImportShapefile.importShapefile(shapefile, false, ds, false); ODLTableReadOnly table = ds.getTableAt(0); if (table.getRowCount() == 0 || processingApi.isCancelled()) { return; } int geomCol = TableUtils.findColumnIndx(table, ODLColumnType.GEOM); int nrows = table.getRowCount(); // Create map to store positions in the output file LargeList<ShapeIndex> indices = new LargeList<>(); for (int row = 0; row < nrows; row++) { indices.add(new ShapeIndex(row, tileFactoryInfo.getMaximumZoomLevel(), (ODLGeom) table.getValueAt(row, geomCol))); } // Get full WGS84 geometries into collection of PendingWrites WKBWriter geomWriter = new WKBWriter(); LargeList<PendingWrite> pws = new LargeList<>(); for (int row = 0; row < nrows; row++) { Geometry g = ((ODLGeomImpl) table.getValueAt(row, geomCol)).getJTSGeometry(); byte[] bytes = geomWriter.write(g); pws.add(new PendingWrite(indices.get(row), bytes)); } if(processingApi.isCancelled()){ return; } // write full geometries QuadWriter quadWriter = new QuadWriter(tmpFile); quadWriter.add(pws, null, -1); if(processingApi.isCancelled()){ return; } // Create executor service ExecutorService executorService = Executors.newFixedThreadPool(nbThreads); // Loop over zoom levels for (int zoom = tileFactoryInfo.getMinimumZoomLevel(); zoom <= tileFactoryInfo.getMaximumZoomLevel(); zoom++) { postStatusMessage("ODLRG builder - processing zoom level " + zoom + " with " + ((long)tileFactoryInfo.getLongitudeDegreeWidthInPixels(zoom)) + " pixels/degree"); // Create converter for this zoom level TransformGeomToWorldBitmap mathTransform = createTransform(zoom); // Get a list and size it to store the results LargeList<PendingWrite> pendingWrites = new LargeList<>(); for (int row = 0; row < nrows; row++) { pendingWrites.add(null); } // create a per-thread processor and then invoke all RowAllocator allocator = new RowAllocator(nrows); ArrayList<RowProcessor> processors = new ArrayList<>(); for (int i = 0; i < nbThreads; i++) { processors.add(new RowProcessor(table, geomCol, indices, zoom, mathTransform, allocator,"ODLRG builder - processing zoom level " + zoom + "", pendingWrites)); } List<Future<Void>> futures = executorService.invokeAll(processors); for (Future<Void> future : futures) { future.get(); } // Process all pending writes LargeList<PendingWrite> nonNulls = new LargeList<>(); for (PendingWrite pw : pendingWrites) { if (pw != null) { nonNulls.add(pw); } } quadWriter.add(nonNulls, tileFactoryInfo, zoom); if(processingApi.isCancelled()){ return; } } // shutdown executor service executorService.shutdown(); // create final file quadWriter.finish(isNOLPL,indices, outfile); // try loading it validateFinalFile(); } catch (Exception e) { throw new RuntimeException(e); } } static class PendingWrite implements Comparable<PendingWrite> { final ShapeIndex index; byte[]bjsonBytes; final byte[] geomBytes; Point centroid; PendingWrite(ShapeIndex index, byte[] bytes) { this.index = index; this.geomBytes = bytes; } @Override public int compareTo(PendingWrite o) { return Integer.compare(index.rowNb, o.index.rowNb); } } // /** // * @param indices // * @throws FileNotFoundException // * @throws IOException // */ // private void createMergedFile(LargeList<ShapeIndex> indices) throws FileNotFoundException, IOException { // // // Create header file // FileOutputStream outFos = new FileOutputStream(outfile); // FileChannel outChannel = outFos.getChannel(); // writeIndex(indices, outChannel); // // // check predicted size against file position, remembering to flush the buffer // long predictedHeaderSize = getPredictedHeaderIndexSize(indices); // if(outChannel.position() != predictedHeaderSize){ // throw new RuntimeException("Incorrect header size written"); // } // // // Now append other file, validating the positions as we append // FileInputStream fis = new FileInputStream(tmpFile); // //FileChannel inChannel = fis.getChannel(); // BufferedInputStream bis = new BufferedInputStream(fis); // DataInputStream dis = new DataInputStream(bis); // // while(true){ // // get output current position // long currentOutPos = outChannel.position(); // // // read first byte to see if we have an entry left // int b1 =bis.read(); // if(b1==-1){ // break; // } // // // read the next 7 bytes of the id and turn into the long // ByteBuffer buffer = ByteBuffer.allocate(8); // buffer.put((byte)b1); // for(int i =0 ; i < 7 ; i++){ // buffer.put((byte)bis.read()); // } // buffer.flip(); // long id = buffer.getLong(); // // // check id is valid // if(id >= indices.size() || id<0){ // throw new RuntimeException("Invalid id"); // } // // // check this position is known in the index // ShapeIndex indx = indices.get(id); // boolean found= currentOutPos == indx.originalWGS84GeomPosition + predictedHeaderSize; // for(int zoom=0 ; zoom < indx.positions.length && !found ; zoom++){ // found = currentOutPos == indx.positions[zoom] + predictedHeaderSize; // } // // if(!found){ // throw new RuntimeException("Position not found in geometry"); // } // // // read the geometry bytes in // int nbytes = dis.readInt(); // byte [] bytes = new byte[nbytes]; // dis.read(bytes); // // // write back out again // ROGUtils.writeGeom(id, bytes, outChannel); // } // // // // close files // dis.close(); // outFos.flush(); // outFos.close(); // // } /** * @throws IOException */ public void validateFinalFile() { final QuadLoader loader = new QuadLoader(outfile); try { // System.out.println("Validating final output file"); List<ODLRenderOptimisedGeom> geoms = loader.readObjects(); for (int i = 0; i < geoms.size(); i++) { ODLRenderOptimisedGeom geom = geoms.get(i); for (int zoom = -1; zoom <= tileFactoryInfo.getMaximumZoomLevel(); zoom++) { int[] pos = geom.getFilePosition(zoom); if (pos != null && pos[0] >= 0) { loader.loadGeometry(i, pos[0], pos[1]); } } if(i%100==0){ postStatusMessage("ODLRG builder - validated " + (i+1) + " object(s) across all zoom levels"); } if(processingApi.isCancelled()){ loader.close(); return; } } } catch (Exception e) { throw new RuntimeException(e); } finally{ try { loader.close(); } catch (Exception e2) { throw new RuntimeException(e2); } } } // /** // * @param indices // * @param out // * @throws IOException // */ // private void writeIndex(LargeList<ShapeIndex> indices, FileChannel outChannel) throws IOException { // DataOutputStream out = new DataOutputStream(new BufferedOutputStream(Channels.newOutputStream(outChannel))); // // // Get header size // long predictedHeaderSize = getPredictedHeaderIndexSize(indices); // // // Write version and number of entries to header // long checkHeaderSize = 0; // out.writeInt(AppConstants.RENDER_GEOMETRY_FILE_VERSION); // checkHeaderSize += 4; // out.writeInt(indices.size()); // checkHeaderSize += 4; // // // Write each entry to header // for (int i = 0; i < indices.size(); i++) { // // // write row number / id // ShapeIndex indx = indices.get(i); // out.writeLong(indx.rowNb); // checkHeaderSize += 8; // // // write total points count // out.writeInt(indx.nbPointsFullGeometry); // checkHeaderSize += 4; // // // write count by shape // out.writeInt(indx.pointsCount); // out.writeInt(indx.linestringsCount); // out.writeInt(indx.polysCount); // checkHeaderSize += 3 * 4; // // // write bounds // out.writeDouble(indx.getWgsBounds().getMinX()); // out.writeDouble(indx.getWgsBounds().getMinY()); // out.writeDouble(indx.getWgsBounds().getWidth()); // out.writeDouble(indx.getWgsBounds().getHeight()); // checkHeaderSize += 4 * 8; // // // write latitude // out.writeDouble(indx.getWgsCentroid().getLongitude()); // out.writeDouble(indx.getWgsCentroid().getLatitude()); // checkHeaderSize += 2 * 8; // // // write full geometry position // out.writeLong(predictedHeaderSize + indx.getOriginalWGS84GeomPosition()); // checkHeaderSize += 8; // // // write array size // out.writeByte(indx.positions.length); // checkHeaderSize++; // // // write position array // for (int j = 0; j < indx.positions.length; j++) { // long val = indx.positions[j]; // if (val >= 0) { // val += predictedHeaderSize; // } // out.writeLong(val); // checkHeaderSize += 8; // } // } // if (checkHeaderSize != predictedHeaderSize) { // out.close(); // throw new RuntimeException("Incorrect header size"); // } // // if (checkHeaderSize < Integer.MAX_VALUE && checkHeaderSize != out.size()) { // out.close(); // throw new RuntimeException("Incorrect header size"); // } // // // ensure whole header is written to physical file // out.flush(); // } // // /** // * @param indices // * @return // */ // private long getPredictedHeaderIndexSize(LargeList<ShapeIndex> indices) { // int entrySize = 8 + 4 + (3 * 4) + (4 * 8) + (2 * 8) + 8 + 1 + indices.get(0).positions.length * 8; // long predictedHeaderSize = 4L + 4 + entrySize * indices.size(); // return predictedHeaderSize; // } // /** // * @param rf // * @param pos // */ // private void testReadGeometry(RandomAccessFile rf, long pos) { // try { // final InputStream in = Channels.newInputStream(rf.getChannel().position(pos)); // Geometry g = new WKBReader().read(new InStream() { // // @Override // public void read(byte[] buf) throws IOException { // in.read(buf); // } // }); // if(g==null){ // throw new RuntimeException(); // } // } catch (Exception e) { // throw new RuntimeException(e); // } // } private TransformGeomToWorldBitmap createTransform(final int zoom) { LatLongToScreen converter = new LatLongToScreenImpl() { @Override public Rectangle2D getViewportWorldBitmapScreenPosition() { return null; } @Override public Point2D getWorldBitmapPixelPosition(LatLong latLong) { Point2D point = GeoUtil.getBitmapCoordinate(new GeoPosition(latLong.getLatitude(), latLong.getLongitude()), zoom, tileFactoryInfo); return point; } @Override public LatLong getLongLat(double pixelX, double pixelY) { throw new UnsupportedOperationException(); } @Override public Object getZoomHashmapKey() { return zoom; } @Override public int getZoomForObjectFiltering() { return zoom; } }; TransformGeomToWorldBitmap mathTransform = new TransformGeomToWorldBitmap(converter); return mathTransform; } }