package se.kodapan.osm.sweden.ext.se.posten.postnummer.local;
import com.vividsolutions.jts.geom.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import se.kodapan.osm.parser.xml.OsmXmlParserException;
import se.kodapan.osm.parser.xml.instantiated.InstantiatedOsmXmlParser;
import se.kodapan.osm.domain.*;
import se.kodapan.osm.util.distance.ArcDistance;
import se.kodapan.osm.jts.AdjacentClassVoronoiClusterer;
import se.kodapan.osm.jts.EnvelopeGrid;
import se.kodapan.osm.jts.JtsSerializer;
import se.kodapan.osm.jts.LineInterpolation;
import se.kodapan.osm.services.overpass.Overpass;
import se.kodapan.osm.services.overpass.OverpassException;
import se.kodapan.osm.services.overpass.OverpassUtils;
import se.kodapan.osm.sweden.OsmSweden;
import se.kodapan.osm.sweden.Sweden;
import se.kodapan.osm.sweden.util.Scored;
import se.kodapan.osm.xml.OsmXmlWriter;
import java.io.*;
import java.net.URLEncoder;
import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author kalle
* @since 2013-08-31 6:29 PM
*/
public class PostortPolygonsExtractorFactory2 {
public static void main(String[] args) throws Exception {
Overpass overpass = new Overpass();
overpass.setServerURL("http://192.168.1.3/api/interpreter");
overpass.setUserAgent("osm.kodapan.se PostortPolygonsExtractorFactory");
overpass.open();
OsmSweden.getInstance().open();
try {
PostortPolygonsExtractorFactory2 factory = new PostortPolygonsExtractorFactory2(OsmSweden.getInstance().getLocalPosten(), overpass);
factory.init();
factory.firstIteration();
Map<Postort, List<Polygon>> secondIteration = factory.secondIteration();
Map<Postort, List<Polygon>> thirdIteration = factory.thirdIteration(secondIteration);
System.currentTimeMillis();
} finally {
overpass.close();
OsmSweden.getInstance().close();
}
}
private static Logger log = LoggerFactory.getLogger(PostortPolygonsExtractorFactory2.class);
private GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(PrecisionModel.FLOATING));
private File path = new File("target/postort");
private Root localPosten;
private Overpass overpass;
private OverpassUtils overpassUtils;
private int numberOfThreads = 16;
public PostortPolygonsExtractorFactory2(Root localPosten, Overpass overpass) {
this.localPosten = localPosten;
this.overpass = overpass;
this.overpassUtils = new OverpassUtils(overpass);
}
private boolean useBorderland = false;
/**
* 4500 points rather than 12000 points, making the voronoi result a lot faster to evaluate
*/
private boolean usingSimplifiedSwedishBorders = true;
private Postort borderland;
private double borderlandInterpolationKilometers = 1;
private se.kodapan.osm.domain.root.Root sweden;
/**
* Loads the Swedish border, etc. Things used by all iterations.
*/
public void init() throws OsmXmlParserException, OverpassException {
borderland = new Postort();
borderland.setIdentity("borderland");
if (useBorderland) {
log.debug("Downloading and processing the Swedish border.");
if (usingSimplifiedSwedishBorders) {
sweden = new Sweden(overpass).loadSimplifiedBorders();
} else {
sweden = new Sweden(overpass).loadBorders();
}
}
}
/**
* Create postort Voronoi polygons from ref:se:pts:postort in OpenStreetMap.
*/
public Map<Postort, List<Polygon>> firstIteration() {
log.info("First iteration.");
// not really needed, but is nice for demonstration
File path = new File(this.path, "iteration 1");
path.mkdirs();
File voronoiOsmFile = new File(path, "postorter.osm");
File voronoiObjectFile = new File(path, "postorter.object");
Map<Postort, List<Polygon>> voronoi = null;
if (voronoiObjectFile.exists()) {
try {
voronoi = deserializeVoronoi(voronoiObjectFile, borderland, new JtsSerializer(geometryFactory));
} catch (Exception e) {
log.error("Could not load " + voronoiObjectFile.getAbsolutePath(), e);
voronoi = null;
}
}
if (voronoi == null) {
final InstantiatedOsmXmlParser parser = new InstantiatedOsmXmlParser();
final AdjacentClassVoronoiClusterer<Postort> voronoiFactory = new AdjacentClassVoronoiClusterer<Postort>(geometryFactory);
if (useBorderland) {
for (Way way : sweden.getWays().values()) {
for (Coordinate coordinate : new LineInterpolation().interpolate(borderlandInterpolationKilometers, way)) {
voronoiFactory.addCoordinate(borderland, coordinate);
}
}
}
for (final Postort postort : localPosten.getPostortByIdentity().values()) {
for (OsmObject osmObject : postort.getReferencedObjects()) {
osmObject.accept(new OsmObjectVisitor<Void>() {
@Override
public Void visit(Node node) {
if (!node.isLoaded()) {
try {
overpassUtils.loadNode(parser, node.getId());
} catch (Exception e1) {
throw new RuntimeException(e1);
}
}
voronoiFactory.addCoordinate(postort, node);
return null;
}
@Override
public Void visit(Way way) {
log.warn("Skipping import of way in postort " + postort.getIdentity());
return null;
}
@Override
public Void visit(Relation relation) {
log.warn("Skipping import relation way in postort " + postort.getIdentity());
return null;
}
});
}
}
try {
voronoi = voronoiFactory.build();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
try {
serializeVoronoi(voronoiObjectFile, new JtsSerializer(geometryFactory), voronoi);
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
}
try {
OsmXmlWriter osmxml = new OsmXmlWriter(new OutputStreamWriter(new FileOutputStream(voronoiOsmFile), "utf8"));
AdjacentClassVoronoiClusterer.OsmRootFactory<Postort> factory = new AdjacentClassVoronoiClusterer.OsmRootFactory<Postort>() {
private String source = "Voronoi of points with ref:se:pts:postort set. osm@kodapan.se";
@Override
public void setNode(Node node, Postort postort, List<Polygon> geometries, Polygon geometry, Coordinate coordinate) {
node.setTag("note", "Swedish postort border point. Might be inaccurate.");
node.setTag("source", source);
}
@Override
public void setWay(Way way, Postort postort, List<Polygon> geometries, Polygon geometry) {
way.setTag("note", "Swedish postort border. Might be inaccurate.");
way.setTag("source", source);
}
@Override
public void setClassTypeInstanceMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries) {
relation.setTag("source", source);
relation.setTag("ref:se:pts:postort", postort.getIdentity());
}
@Override
public void setGeometryMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries, Polygon geometry) {
relation.setTag("note", "A complex polygon describing part of the Swedish postal town " + postort.getIdentity());
relation.setTag("source", source);
}
};
osmxml.write(factory.factory(voronoi));
osmxml.close();
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
return voronoi;
}
/**
* Classify a grid around each point ref:se:pts:postort in OpenStreetMap,
* adding a point for found postort at centroid of classification area.
*/
public Map<Postort, List<Polygon>> secondIteration() {
log.info("Second iteration.");
File path = new File(this.path, "iteration 2");
path.mkdirs();
File voronoiOsmFile = new File(path, "postorter.osm");
File voronoiObjectFile = new File(path, "postorter.object");
final File previousPointsPath = new File(path, "search points");
previousPointsPath.mkdirs();
final AdjacentClassVoronoiClusterer<Postort> voronoiFactory = new AdjacentClassVoronoiClusterer<Postort>(geometryFactory);
if (useBorderland) {
for (Way way : sweden.getWays().values()) {
for (Coordinate coordinate : new LineInterpolation().interpolate(borderlandInterpolationKilometers, way)) {
voronoiFactory.addCoordinate(borderland, coordinate);
}
}
}
Map<Postort, List<Polygon>> voronoi = null;
if (voronoiObjectFile.exists()) {
try {
voronoi = deserializeVoronoi(voronoiObjectFile, borderland, new JtsSerializer(geometryFactory));
} catch (Exception e) {
log.error("Could not load " + voronoiObjectFile.getAbsolutePath(), e);
voronoi = null;
}
}
if (voronoi == null) {
final Set<EnvelopeGrid.Envelope> envelopesAlreadyClassified = loadClassifications(previousPointsPath, voronoiFactory);
log.debug("classify in a grid around each postort ref:se:pts:postort point");
List<Postort> postorter = new ArrayList<Postort>(getPostorterOrderedByDistanceFromPoint(stockholmLatLon, localPosten.getPostortByIdentity().values()));
//todo debug remove
// postorter.clear();
// postorter.add(localPosten.getPostortByIdentity().get("ÄLVSJÖ"));
//todo debug remove
final AtomicBoolean stopThreads = new AtomicBoolean(false);
for (final Postort postort : postorter) {
log.info("Processing " + postort.getIdentity());
for (OsmObject osmObject : postort.getReferencedObjects()) {
osmObject.accept(new OsmObjectVisitor<Void>() {
private ArcDistance distance = new ArcDistance();
@Override
public Void visit(final Node node) {
voronoiFactory.addCoordinate(postort, node);
if (stopThreads.get()) {
return null;
}
EnvelopeGrid grid = new EnvelopeGrid(10000, node.getLatitude(), node.getLongitude(), 20, 20);
List<EnvelopeGrid.Envelope> envelopes = new ArrayList<EnvelopeGrid.Envelope>();
for (EnvelopeGrid.Envelope[] rows : grid.getGrid()) {
Collections.addAll(envelopes, rows);
}
Collections.sort(envelopes, new Comparator<EnvelopeGrid.Envelope>() {
@Override
public int compare(EnvelopeGrid.Envelope o1, EnvelopeGrid.Envelope o2) {
double distance1 = distance.calculate(node.getLatitude(), node.getLongitude(), o1.getNorthLatitude(), o1.getWestLongitude());
double distance2 = distance.calculate(node.getLatitude(), node.getLongitude(), o2.getNorthLatitude(), o2.getWestLongitude());
return Double.compare(distance1, distance2);
}
});
final ConcurrentLinkedQueue<EnvelopeGrid.Envelope> queue = new ConcurrentLinkedQueue<EnvelopeGrid.Envelope>(envelopes);
Thread[] threads = new Thread[numberOfThreads];
String suffix = String.valueOf(System.currentTimeMillis());
for (int threadIndex = 0; threadIndex < numberOfThreads; threadIndex++) {
final File logFile;
try {
logFile = new File(previousPointsPath, URLEncoder.encode(postort.getIdentity(), "utf8") + "." + threadIndex + "." + suffix);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
threads[threadIndex] = new Thread(new Runnable() {
@Override
public void run() {
final ObjectOutputStream searchPointsLog;
{
ObjectOutputStream objectOutputStream;
try {
objectOutputStream = new ObjectOutputStream(new FileOutputStream(logFile));
} catch (IOException e) {
log.error("Could not create search point log!", e);
objectOutputStream = null;
}
searchPointsLog = objectOutputStream;
}
try {
EnvelopeGrid.Envelope envelope;
while (!stopThreads.get() && (envelope = queue.poll()) != null) {
if (envelopesAlreadyClassified.contains(envelope)) {
log.debug("Skipping envelope in " + postort.getIdentity() + " since it already was loaded. " + envelope);
continue;
}
List<Scored<Postort>> classifications;
try {
classifications = classifyPostort(envelope);
} catch (Exception e) {
stopThreads.set(true);
log.error("Caught exception in worker thread. Stopping all.", e);
return;
}
new PostortsClassificationSelector<EnvelopeGrid.Envelope>(envelope) {
@Override
public void select(Postort postort) {
voronoiFactory.addCoordinate(postort, node);
if (searchPointsLog != null) {
// save envelope as classified
try {
searchPointsLog.writeBoolean(true);
if (postort == null) {
searchPointsLog.writeObject(null);
} else {
searchPointsLog.writeObject(postort.getIdentity());
}
searchPointsLog.writeObject(getMetaData());
searchPointsLog.flush();
} catch (Exception e) {
log.error("Error writing to voronoi search log!", e);
}
}
}
}.select(classifications);
}
} finally {
if (searchPointsLog != null) {
try {
searchPointsLog.writeBoolean(false);
searchPointsLog.close();
} catch (Exception e) {
log.error("Error while closing postort classification log " + logFile.getName(), e);
}
}
}
}
});
threads[threadIndex].setDaemon(true);
threads[threadIndex].setName("Scan grid around ref:se:pts:postort thread #" + threadIndex);
threads[threadIndex].start();
}
Thread debugThread = new Thread(new Runnable() {
@Override
public void run() {
while (true) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
log.error("Caught exception while sleeping", e);
break;
}
if (stopThreads.get()) {
break;
}
if (queue.isEmpty()) {
break;
}
log.info(queue.size() + " envelopes left to process in grid around references of " + postort.getIdentity() + "...");
}
}
});
debugThread.setDaemon(true);
debugThread.setName("Scan grid around references debug thread");
debugThread.start();
for (Thread thread : threads) {
try {
thread.join();
} catch (InterruptedException ie) {
log.error("Caught exception when joining worker threads. Breaks out of joining threads.", ie);
break;
}
}
// end visitor
return null;
}
@Override
public Void visit(Way way) {
log.warn("Skipping import of way in postort " + postort.getIdentity());
return null;
}
@Override
public Void visit(Relation relation) {
log.warn("Skipping import of relation in postort " + postort.getIdentity());
return null;
}
});
}
if (stopThreads.get()) {
break;
}
}
try {
voronoi = voronoiFactory.build();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
try {
serializeVoronoi(voronoiObjectFile, new JtsSerializer(geometryFactory), voronoi);
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
try {
OsmXmlWriter osmxml = new OsmXmlWriter(new OutputStreamWriter(new FileOutputStream(voronoiOsmFile), "utf8"));
AdjacentClassVoronoiClusterer.OsmRootFactory<Postort> factory = new AdjacentClassVoronoiClusterer.OsmRootFactory<Postort>() {
private String source = "Voronoi of points with ref:se:pts:postort set, tested against data available in Postnummersystemet. osm@kodapan.se";
@Override
public void setNode(Node node, Postort postort, List<Polygon> geometries, Polygon geometry, Coordinate coordinate) {
node.setTag("note", "Swedish postort border point. Might be inaccurate.");
node.setTag("source", source);
}
@Override
public void setWay(Way way, Postort postort, List<Polygon> geometries, Polygon geometry) {
way.setTag("note", "Swedish postort border. Might be inaccurate.");
way.setTag("source", source);
}
@Override
public void setClassTypeInstanceMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries) {
relation.setTag("note", postort.getIdentity() + "-group with " + voronoiFactory.getCoordinatesByClass().get(postort).size() + " points describing the Swedish postal town");
relation.setTag("source", source);
relation.setTag("ref:se:pts:postort", postort.getIdentity());
}
@Override
public void setGeometryMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries, Polygon geometry) {
relation.setTag("note", "A complex polygon describing part of the Swedish postal town " + postort.getIdentity());
relation.setTag("source", source);
}
};
osmxml.write(factory.factory(voronoi));
osmxml.close();
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
}
return voronoi;
}
private Set<EnvelopeGrid.Envelope> loadClassifications(File path, AdjacentClassVoronoiClusterer<Postort> voronoiFactory) {
log.info("Load previously clasified points");
final Set<EnvelopeGrid.Envelope> envelopesUsedForClassification = new HashSet<EnvelopeGrid.Envelope>(1000000);
for (File file : path.listFiles(new FileFilter() {
@Override
public boolean accept(File file) {
return file.isFile();
}
})) {
try {
log.debug("Processing " + file.getAbsolutePath());
ObjectInputStream in = new ObjectInputStream(new FileInputStream(file));
try {
while ((in.readBoolean())) {
String postort = (String) in.readObject();
EnvelopeGrid.Envelope envelope = (EnvelopeGrid.Envelope) in.readObject();
if (envelope.getEastLongitude() > 25 || envelope.getWestLongitude() < 10 || envelope.getNorthLatitude() > 70 || envelope.getSouthLatitude() < 55) {
log.error("Envelope bounds out of sweden!");
} else {
if (envelopesUsedForClassification.add(envelope)) {
if (postort != null) {
double longitude = (envelope.getEastLongitude() + envelope.getWestLongitude()) / 2;
double latitude = (envelope.getNorthLatitude() + envelope.getSouthLatitude()) / 2;
voronoiFactory.addCoordinate(localPosten.getPostortByIdentity().get(postort), longitude, latitude);
}
}
}
}
} catch (EOFException eof) {
log.debug("EOF ignored", eof);
} finally {
in.close();
}
} catch (Exception e) {
log.error("Ignoring exception from processing " + file.getAbsolutePath(), e);
}
}
log.info(voronoiFactory.countCoordinates() + " coordinates loaded from known data.");
return envelopesUsedForClassification;
}
private double[] malmöLatLon = new double[]{55.60123861794095, 13.009185791015625};
private double[] alvsjoLatLon= new double[]{59.276, 18};
private double[] stockholmLatLon = new double[]{59.324, 18.07};
final double[] halmstadLatLon = new double[]{56.66, 12.84};
final double[] göteborgLatLon = new double[]{57.7, 12};
final double[] fjbLatLon = new double[]{58.61, 11.28};
final double[] kirunaLatLon = new double[]{67.8, 20.2};
private List<Postort> getPostorterOrderedByDistanceFromPoint(final double[] centroidLatLon, Collection<Postort> input) {
final ArcDistance distance = new ArcDistance();
List<Postort> postorter = new ArrayList<Postort>(input);
Collections.sort(postorter, new Comparator<Postort>() {
@Override
public int compare(Postort o1, Postort o2) {
Node node1 = null;
Node node2 = null;
if (!o1.getReferencedObjects().isEmpty()) {
node1 = (Node) o1.getReferencedObjects().iterator().next();
}
if (!o2.getReferencedObjects().isEmpty()) {
node2 = (Node) o2.getReferencedObjects().iterator().next();
}
if (node1 == null && node2 == null) {
return 0;
} else if (node1 == null && node2 != null) {
return 1;
} else if (node2 == null && node1 != null) {
return -1;
} else {
double distance1 = distance.calculate(centroidLatLon[0], centroidLatLon[1], node1.getY(), node1.getX());
double distance2 = distance.calculate(centroidLatLon[0], centroidLatLon[1], node2.getY(), node2.getX());
return Double.compare(distance1, distance2);
}
}
});
return postorter;
}
private <E> List<Map.Entry<E, List<Polygon>>> getVoronoiOrderedByDistanceFromPoint(final double[] centroidLatLon, Map<E, List<Polygon>> input) {
final ArcDistance distance = new ArcDistance();
List<Map.Entry<E, List<Polygon>>> ordered = new ArrayList<Map.Entry<E, List<Polygon>>>(input.entrySet());
Collections.sort(ordered, new Comparator<Map.Entry<E, List<Polygon>>>() {
@Override
public int compare(Map.Entry<E, List<Polygon>> o1, Map.Entry<E, List<Polygon>> o2) {
Coordinate coordinate1 = o1.getValue().get(0).getCentroid().getCoordinate();
Coordinate coordinate2 = o2.getValue().get(0).getCentroid().getCoordinate();
double distance1 = distance.calculate(centroidLatLon[0], centroidLatLon[1], coordinate1.y, coordinate1.x);
double distance2 = distance.calculate(centroidLatLon[0], centroidLatLon[1], coordinate2.y, coordinate2.x);
return Double.compare(distance1, distance2);
}
});
return ordered;
}
/**
* Follow borders of postort polygons
* and classify at more detailed level between neighbouring postorter.
*/
public Map<Postort, List<Polygon>> thirdIteration(Map<Postort, List<Polygon>> inputPostortAndPolygons) {
log.info("Third iteration.");
final ArcDistance distance = new ArcDistance();
File path = new File(this.path, "iteration 3");
path.mkdirs();
File voronoiOsmFile = new File(path, "postorter.osm");
File voronoiObjectFile = new File(path, "postorter.object");
final File previousPointsPath = new File(path, "search points");
previousPointsPath.mkdirs();
final AdjacentClassVoronoiClusterer<Postort> voronoiFactory = new AdjacentClassVoronoiClusterer<Postort>(geometryFactory);
if (useBorderland) {
for (Way way : sweden.getWays().values()) {
for (Coordinate coordinate : new LineInterpolation().interpolate(borderlandInterpolationKilometers, way)) {
voronoiFactory.addCoordinate(borderland, coordinate);
}
}
}
final Set<EnvelopeGrid.Envelope> envelopesAlreadyClassified = loadClassifications(previousPointsPath, voronoiFactory);
List<Map.Entry<Postort, List<Polygon>>> ordered = getVoronoiOrderedByDistanceFromPoint(alvsjoLatLon, inputPostortAndPolygons);
// todo debug remove
// ordered.clear();
// Map<Postort, List<Polygon>> hack = new HashMap<Postort, List<Polygon>>();
// hack.put(localPosten.getPostortByIdentity().get("ÄLVSJÖ"), inputPostortAndPolygons.get(localPosten.getPostortByIdentity().get("ÄLVSJÖ")));
// ordered = new ArrayList<Map.Entry<Postort, List<Polygon>>>(hack.entrySet());
// todo debug remove
final AtomicBoolean stopThreads = new AtomicBoolean(false);
for (Map.Entry<Postort, List<Polygon>> postortAndPolygons : ordered) {
final Postort postort = postortAndPolygons.getKey();
if (borderland.equals(postort)) {
continue;
}
// log.info("Searching on the borders of " + postort.getIdentity());
List<Coordinate> interpolatedPolygonPoints = new ArrayList<Coordinate>();
for (Polygon polygon : postortAndPolygons.getValue()) {
Coordinate[] coordinates = polygon.getCoordinates();
for (int i = 1; i < coordinates.length; i++) {
Coordinate previousCoordinate = coordinates[i - 1];
Coordinate coordinate = coordinates[i];
interpolatedPolygonPoints.addAll(new LineInterpolation().interpolate(5, previousCoordinate, coordinate));
}
}
log.info("Searching on the borders of " + postort.getIdentity() + " on " + interpolatedPolygonPoints.size() + " points.");
final ConcurrentLinkedQueue<EnvelopeGrid.Envelope> queue = new ConcurrentLinkedQueue<EnvelopeGrid.Envelope>();
for (final Coordinate coordinate : interpolatedPolygonPoints) {
voronoiFactory.addCoordinate(postort, coordinate);
EnvelopeGrid grid = new EnvelopeGrid(2500, coordinate.y, coordinate.x, 5, 5);
List<EnvelopeGrid.Envelope> envelopes = new ArrayList<EnvelopeGrid.Envelope>();
EnvelopeGrid.Envelope[][] rows = grid.getGrid();
for (EnvelopeGrid.Envelope[] columns : rows) {
Collections.addAll(envelopes, columns);
}
Collections.sort(envelopes, new Comparator<EnvelopeGrid.Envelope>() {
@Override
public int compare(EnvelopeGrid.Envelope o1, EnvelopeGrid.Envelope o2) {
double distance1 = distance.calculate(coordinate.y, coordinate.x, o1.getNorthLatitude(), o1.getWestLongitude());
double distance2 = distance.calculate(coordinate.y, coordinate.x, o2.getNorthLatitude(), o2.getWestLongitude());
return Double.compare(distance1, distance2);
}
});
queue.addAll(envelopes);
}
{
Thread[] threads = new Thread[numberOfThreads];
String suffix = String.valueOf(System.currentTimeMillis());
for (int threadIndex = 0; threadIndex < numberOfThreads; threadIndex++) {
final File logFile;
try {
logFile = new File(previousPointsPath, URLEncoder.encode(postort.getIdentity(), "utf8") + "." + threadIndex + "." + suffix);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
threads[threadIndex] = new Thread(new Runnable() {
@Override
public void run() {
ObjectOutputStream objectOutputStream = null;
try {
objectOutputStream = new ObjectOutputStream(new FileOutputStream(logFile));
} catch (IOException e) {
log.error("Could not create search point log!", e);
objectOutputStream = null;
}
final ObjectOutputStream searchPointsLog = objectOutputStream;
EnvelopeGrid.Envelope envelope;
while (!stopThreads.get() && (envelope = queue.poll()) != null) {
if (envelopesAlreadyClassified.contains(envelope)) {
log.debug("Skipping envelope in " + postort.getIdentity() + " since it already was loaded. " + envelope);
continue;
}
List<Scored<Postort>> classifications;
try {
classifications = classifyPostort(envelope);
} catch (Exception e) {
stopThreads.set(true);
log.error("Caught exception in worker thread. Stopping all.", e);
return;
}
new PostortsClassificationSelector<EnvelopeGrid.Envelope>(envelope) {
@Override
public void select(Postort postort) {
voronoiFactory.addCoordinate(postort, getMetaData().getCentroidLongitude(), getMetaData().getCentroidLatitude());
if (searchPointsLog != null) {
// save envelope as classified
try {
searchPointsLog.writeBoolean(true);
if (postort == null) {
searchPointsLog.writeObject(null);
} else {
searchPointsLog.writeObject(postort.getIdentity());
}
searchPointsLog.writeObject(getMetaData());
searchPointsLog.flush();
} catch (Exception e) {
log.error("Error writing to voronoi search log!", e);
}
}
}
}.select(classifications);
}
if (searchPointsLog != null) {
try {
searchPointsLog.writeBoolean(false);
searchPointsLog.close();
} catch (Exception e) {
log.error("Error while closing postort classification log " + logFile.getName(), e);
}
}
}
});
threads[threadIndex].setDaemon(true);
threads[threadIndex].setName("Scan grid around border thread #" + threadIndex);
threads[threadIndex].start();
}
Thread debugThread = new Thread(new Runnable() {
@Override
public void run() {
while (true) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
log.error("Caught exception while sleeping", e);
break;
}
if (stopThreads.get()) {
break;
}
if (queue.isEmpty()) {
break;
}
log.info(queue.size() + " envelopes left to process in borders around " + postort.getIdentity() + "...");
}
}
});
debugThread.setDaemon(true);
debugThread.setName("Scan grid around border debug thread");
debugThread.start();
for (Thread thread : threads) {
try {
thread.join();
} catch (InterruptedException ie) {
log.error("Caught exception when joining worker threads. Breaks out of joining threads.", ie);
break;
}
}
if (stopThreads.get()) {
break;
}
}
if (stopThreads.get()) {
break;
}
}
Map<Postort, List<Polygon>> voronoi;
try {
voronoi = voronoiFactory.build();
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
try {
serializeVoronoi(voronoiObjectFile, new JtsSerializer(geometryFactory), voronoi);
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
try {
OsmXmlWriter osmxml = new OsmXmlWriter(new OutputStreamWriter(new FileOutputStream(voronoiOsmFile), "utf8"));
AdjacentClassVoronoiClusterer.OsmRootFactory<Postort> factory = new AdjacentClassVoronoiClusterer.OsmRootFactory<Postort>() {
private String source = "Voronoi of points with ref:se:pts:postort set, tested against data available in Postnummersystemet. osm@kodapan.se";
@Override
public void setNode(Node node, Postort postort, List<Polygon> geometries, Polygon geometry, Coordinate coordinate) {
node.setTag("note", "Swedish postort border point. Might be inaccurate.");
node.setTag("source", source);
}
@Override
public void setWay(Way way, Postort postort, List<Polygon> geometries, Polygon geometry) {
way.setTag("note", "Swedish postort border. Might be inaccurate.");
way.setTag("source", source);
}
@Override
public void setClassTypeInstanceMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries) {
relation.setTag("note", postort.getIdentity() + "-group with " + voronoiFactory.getCoordinatesByClass().get(postort).size() + " points describing the Swedish postal town");
relation.setTag("source", source);
relation.setTag("ref:se:pts:postort", postort.getIdentity());
}
@Override
public void setGeometryMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries, Polygon geometry) {
relation.setTag("note", "A complex polygon describing part of the Swedish postal town " + postort.getIdentity());
relation.setTag("source", source);
}
};
osmxml.write(factory.factory(voronoi));
osmxml.close();
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
return voronoi;
}
public abstract class PostortsClassificationSelector<MetaData> {
public abstract void select(Postort postort);
private MetaData metaData;
protected PostortsClassificationSelector(MetaData metaData) {
this.metaData = metaData;
}
public MetaData getMetaData() {
return metaData;
}
public void select(List<Scored<Postort>> postortScores) {
if (!postortScores.isEmpty()) {
Postort postort = postortScores.get(0).getObject();
if (postortScores.size() == 1 && postortScores.get(0).getScore() >= 6) {
select(postort);
} else if (postortScores.size() > 1 && postortScores.get(0).getScore() >= 10
&& postortScores.get(1).getScore() < postortScores.get(0).getScore() / 2.5d) {
// todo filter in distance to known postorter from the centroid
select(postort);
} else {
select((Postort) null);
}
} else {
select((Postort) null);
}
}
}
public List<Scored<Postort>> classifyPostort(EnvelopeGrid.Envelope column) throws OverpassException, OsmXmlParserException {
InstantiatedOsmXmlParser parser = new InstantiatedOsmXmlParser();
overpassUtils.loadEnvelope(parser, column.getSouthLatitude(), column.getWestLongitude(), column.getNorthLatitude(), column.getEastLongitude());
Set<Way> streetWays = new HashSet<Way>();
Set<Way> houseNumberWays = new HashSet<Way>();
Set<Node> houseNumberNodes = new HashSet<Node>();
for (Way way : parser.getRoot().getWays().values()) {
if (way.getTag("highway") != null && way.getTag("name") != null) {
streetWays.add(way);
} else if (way.getTag("addr:housenumber") != null && way.getTag("addr:street") != null) {
houseNumberWays.add(way);
}
}
for (Node node : parser.getRoot().getNodes().values()) {
if (node.getTag("addr:housenumber") != null && node.getTag("addr:street") != null) {
houseNumberNodes.add(node);
}
}
// gather street names
Set<String> streetNames = new HashSet<String>();
for (OsmObject object : streetWays) {
streetNames.add(object.getTag("name"));
}
for (OsmObject object : houseNumberWays) {
streetNames.add(object.getTag("addr:street"));
}
for (OsmObject object : houseNumberNodes) {
streetNames.add(object.getTag("addr:street"));
}
Map<String, Set<PostnummerSegment>> segmentsPerStreetName = new HashMap<String, Set<PostnummerSegment>>();
Map<Postnummer, Set<PostnummerSegment>> segmentsPerPostnummer = new HashMap<Postnummer, Set<PostnummerSegment>>();
Map<Postort, Set<PostnummerSegment>> segmentsPerPostort = new HashMap<Postort, Set<PostnummerSegment>>();
for (String streetName : streetNames) {
Set<PostnummerSegment> segments = localPosten.getSegmentsByName().get(streetName);
if (segments != null) {
segmentsPerStreetName.put(streetName, segments);
for (PostnummerSegment segment : segments) {
Set<PostnummerSegment> postnummerSegments = segmentsPerPostnummer.get(segment.getPostnummer());
if (postnummerSegments == null) {
postnummerSegments = new HashSet<PostnummerSegment>();
segmentsPerPostnummer.put(segment.getPostnummer(), postnummerSegments);
}
postnummerSegments.add(segment);
}
for (PostnummerSegment segment : segments) {
Set<PostnummerSegment> postortSegments = segmentsPerPostort.get(segment.getPostnummer().getPostort());
if (postortSegments == null) {
postortSegments = new HashSet<PostnummerSegment>();
segmentsPerPostort.put(segment.getPostnummer().getPostort(), postortSegments);
}
postortSegments.add(segment);
}
}
}
List<Scored<Postort>> postortScores = new ArrayList<Scored<Postort>>();
for (Map.Entry<Postort, Set<PostnummerSegment>> postortSegments : segmentsPerPostort.entrySet()) {
Set<String> segmentNames = new HashSet<String>();
for (PostnummerSegment segment : postortSegments.getValue()) {
segmentNames.add(segment.getGatunamn());
}
postortScores.add(new Scored<Postort>(segmentNames.size(), postortSegments.getKey()));
}
Collections.sort(postortScores, Scored.topScoreFirstComparator);
return postortScores;
}
private Map<Postort, List<Polygon>> deserializeVoronoi(File file, Postort borderland, JtsSerializer jts) throws IOException {
Map<Postort, List<Polygon>> classifications = new HashMap<Postort, List<Polygon>>();
ObjectInputStream objectInputStream = new ObjectInputStream(new FileInputStream(file));
while (objectInputStream.readBoolean()) {
String postortsIdentity = objectInputStream.readUTF();
if (postortsIdentity == null) {
System.currentTimeMillis();
}
Postort postort = localPosten.getPostortByIdentity().get(postortsIdentity);
if (postort == null) {
if (borderland.getIdentity().equals(postortsIdentity)) {
postort = borderland;
} else {
System.currentTimeMillis();
throw new NullPointerException();
}
}
List<Polygon> geometries = new ArrayList<Polygon>();
while (objectInputStream.readBoolean()) {
geometries.add(jts.readPolygon(objectInputStream));
}
classifications.put(postort, geometries);
}
objectInputStream.close();
return classifications;
}
private void serializeVoronoi(File file, JtsSerializer jts, Map<Postort, List<Polygon>> geometriesPerPostortRawOsmVoronoi) throws IOException {
ObjectOutputStream objectOutputStream = new ObjectOutputStream(new FileOutputStream(file));
for (Map.Entry<Postort, List<Polygon>> entry : geometriesPerPostortRawOsmVoronoi.entrySet()) {
objectOutputStream.writeBoolean(true);
if (entry.getKey() == null) {
System.currentTimeMillis();
}
objectOutputStream.writeUTF(entry.getKey().getIdentity());
for (Geometry geometry : entry.getValue()) {
objectOutputStream.writeBoolean(true);
Polygon polygon = (Polygon) geometry;
jts.writePolygon(polygon, objectOutputStream);
}
objectOutputStream.writeBoolean(false);
}
objectOutputStream.writeBoolean(false);
objectOutputStream.close();
}
}