package se.kodapan.osm.sweden.ext.se.posten.postnummer.local;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.GeometryFactory;
import com.vividsolutions.jts.geom.Polygon;
import com.vividsolutions.jts.geom.PrecisionModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import se.kodapan.osm.parser.xml.OsmXmlParserException;
import se.kodapan.osm.parser.xml.instantiated.InstantiatedOsmXmlParser;
import se.kodapan.osm.domain.*;
import se.kodapan.osm.util.distance.ArcDistance;
import se.kodapan.osm.jts.AdjacentClassVoronoiClusterer;
import se.kodapan.osm.jts.JtsSerializer;
import se.kodapan.osm.jts.LineInterpolation;
import se.kodapan.osm.services.overpass.Overpass;
import se.kodapan.osm.services.overpass.OverpassException;
import se.kodapan.osm.sweden.Sweden;
import se.kodapan.osm.sweden.ext.se.posten.postnummer.CachedPostenPostnummerService;
import se.kodapan.osm.sweden.util.Scored;
import se.kodapan.osm.xml.OsmXmlWriter;
import java.io.*;
import java.util.*;
/**
* @author kalle
* @since 2013-09-11 12:28 AM
*/
public class PostortPolygonsExtractorFactory3 {
public static void main(String[] args) throws Exception {
// FileSystemCachedOverpass overpass = new FileSystemCachedOverpass(new File("target/postorter/v4/overpass"));
Root localPosten;
CachedPostenPostnummerService posten = new CachedPostenPostnummerService();
posten.setUserAgent("osm.kodapan.se PostortPolygonsExtractorFactory");
posten.setPath(new File("data/CachedPostenPostnummerService"));
posten.open();
try {
// run against live data to fetch most recent postorter references.
Overpass overpass = new Overpass();
// overpass.setServerURL("http://193.0.253.210/api/interpreter");
overpass.setUserAgent("osm.kodapan.se PostortPolygonsExtractorFactory");
overpass.open();
LocalPostenPostnummerService localPostenPostnummerService = new LocalPostenPostnummerService();
localPosten = localPostenPostnummerService.factory(posten, overpass);
overpass.close();
} finally {
posten.close();
}
// run against local overpass to get speed and limitation to sweden.
Overpass overpass = new Overpass();
overpass.setServerURL("http://193.0.253.210/api/interpreter");
overpass.setUserAgent("osm.kodapan.se PostortPolygonsExtractorFactory");
overpass.open();
try {
PostortPolygonsExtractorFactory3 factory = new PostortPolygonsExtractorFactory3(localPosten, overpass);
factory.init();
factory.build();
System.currentTimeMillis();
} finally {
overpass.close();
}
}
private static Logger log = LoggerFactory.getLogger(PostortPolygonsExtractorFactory3.class);
private GeometryFactory geometryFactory = new GeometryFactory(new PrecisionModel(PrecisionModel.FLOATING));
private File path = new File("target/postorter");
private Root localPosten;
private Overpass overpass;
private int numberOfThreads = 16;
public PostortPolygonsExtractorFactory3(Root localPosten, Overpass overpass) {
this.localPosten = localPosten;
this.overpass = overpass;
}
private double matchDistanceKm = 15;
private double mismatchDistanceKm = 150;
private boolean useBorderland = true;
/**
* 4500 points rather than 12000 points, making the voronoi result a lot faster to evaluate
*/
private boolean usingSimplifiedSwedishBorders = false;
private Postort borderland;
private double borderlandInterpolationKilometers = 1;
private se.kodapan.osm.domain.root.Root sweden;
/**
* Loads the Swedish border, etc. Things used by all iterations.
*/
public void init() throws OsmXmlParserException, OverpassException {
borderland = new Postort();
borderland.setIdentity("borderland");
if (useBorderland) {
log.debug("Downloading and processing the Swedish border.");
if (usingSimplifiedSwedishBorders) {
sweden = new Sweden(overpass).loadSimplifiedBorders();
} else {
sweden = new Sweden(overpass).loadBorders();
}
}
}
private ArcDistance distance = new ArcDistance();
public void build() throws Exception {
File path = new File(this.path, "v3");
path.mkdirs();
File voronoiOsmFile = new File(path, "postorter.osm");
File voronoiObjectFile = new File(path, "postorter.object");
final AdjacentClassVoronoiClusterer<Postort> voronoiFactory = new AdjacentClassVoronoiClusterer<Postort>(geometryFactory);
if (useBorderland) {
for (Way way : sweden.getWays().values()) {
for (Coordinate coordinate : new LineInterpolation().interpolate(borderlandInterpolationKilometers, way)) {
voronoiFactory.addCoordinate(borderland, coordinate);
}
}
}
// known ref points
for (final Postort postort : localPosten.getPostortByIdentity().values()) {
for (OsmObject osmObject : postort.getReferencedObjects()) {
osmObject.accept(new OsmObjectVisitor<Void>() {
@Override
public Void visit(Node node) {
voronoiFactory.addCoordinate(postort, node);
return null;
}
@Override
public Void visit(Way way) {
for (Node node : way.getNodes()) {
node.accept(this);
}
return null;
}
@Override
public Void visit(Relation relation) {
for (RelationMembership membership : relation.getMembers()) {
membership.getObject().accept(this);
}
return null;
}
});
}
}
InstantiatedOsmXmlParser parser = new InstantiatedOsmXmlParser();
// streets
{
parser.setRoot(new se.kodapan.osm.domain.root.Root());
parser.parse(overpass.execute("<osm-script>\n" +
"<query type=\"way\">\n" +
"<has-kv k=\"highway\"/>\n" +
"<has-kv k=\"name\"/>\n" +
"</query>\n" +
"<print/>\n" +
"</osm-script>"));
parser.parse(overpass.execute("<osm-script>\n" +
"<query type=\"way\">\n" +
"<has-kv k=\"highway\"/>\n" +
"<has-kv k=\"name\"/>\n" +
"</query>\n" +
"<recurse type=\"way-node\"/>\n" +
"<print/>\n" +
"</osm-script>"));
Set<Postort> postorterChecked = new HashSet<Postort>();
for (final Way way : parser.getRoot().getWays().values()) {
List<Scored<Postort>> distanceToPostorter = new ArrayList<Scored<Postort>>();
Collection<PostnummerSegment> segments = localPosten.getSegmentsByName().get(way.getTag("name"));
if (segments != null) {
postorterChecked.clear();
for (final PostnummerSegment segment : segments) {
if (postorterChecked.add(segment.getPostnummer().getPostort())) {
if (segment.getPostnummer().getPostort().getReferencedObjects() == null || segment.getPostnummer().getPostort().getReferencedObjects().isEmpty()) {
log.error("Missing reference to postort " + segment.getPostnummer().getPostort());
distanceToPostorter.add(new Scored<Postort>(Double.MAX_VALUE, segment.getPostnummer().getPostort()));
} else {
double smallestDistance = Double.MAX_VALUE;
for (OsmObject osmObject : segment.getPostnummer().getPostort().getReferencedObjects()) {
double km = osmObject.accept(new OsmObjectVisitor<Double>() {
@Override
public Double visit(Node currentNode) {
return distance.calculate(currentNode, way.getNodes().get(0));
}
@Override
public Double visit(Way way) {
log.error("Unable to process way");
return Double.MAX_VALUE;
}
@Override
public Double visit(Relation relation) {
log.error("Unable to process relation");
return Double.MAX_VALUE;
}
});
if (km < smallestDistance) {
smallestDistance = km;
if (km == 0) {
break;
}
}
}
distanceToPostorter.add(new Scored<Postort>(smallestDistance, segment.getPostnummer().getPostort()));
}
}
Collections.sort(distanceToPostorter, Scored.lowScoreFirstComparator);
boolean match = false;
if (distanceToPostorter.get(0).getScore() < matchDistanceKm) {
if (distanceToPostorter.size() == 1 || distanceToPostorter.get(1).getScore() > mismatchDistanceKm) {
match = true;
for (Node node : way.getNodes()) {
voronoiFactory.addCoordinate(distanceToPostorter.get(0).getObject(), node);
}
}
}
if (!match) {
System.currentTimeMillis();
}
System.currentTimeMillis();
}
}
}
}
// house number nodes
{
parser.setRoot(new se.kodapan.osm.domain.root.Root());
parser.parse(overpass.execute("<osm-script>\n" +
"<query type=\"node\">\n" +
"<has-kv k=\"addr:housenumber\"/>\n" +
"<has-kv k=\"addr:street\"/>\n" +
"</query>\n" +
"<print/>\n" +
"</osm-script>"));
Set<Postort> postorterChecked = new HashSet<Postort>();
for (final Node node : parser.getRoot().getNodes().values()) {
List<Scored<Postort>> distanceToPostorter = new ArrayList<Scored<Postort>>();
Collection<PostnummerSegment> segments = localPosten.getSegmentsByName().get(node.getTag("addr:street"));
if (segments != null) {
postorterChecked.clear();
for (final PostnummerSegment segment : segments) {
if (postorterChecked.add(segment.getPostnummer().getPostort())) {
if (segment.getPostnummer().getPostort().getReferencedObjects() == null || segment.getPostnummer().getPostort().getReferencedObjects().isEmpty()) {
log.error("Missing reference to postort " + segment.getPostnummer().getPostort());
distanceToPostorter.add(new Scored<Postort>(Double.MAX_VALUE, segment.getPostnummer().getPostort()));
} else {
double smallestDistance = Double.MAX_VALUE;
for (OsmObject osmObject : segment.getPostnummer().getPostort().getReferencedObjects()) {
double km = osmObject.accept(new OsmObjectVisitor<Double>() {
@Override
public Double visit(Node currentNode) {
return distance.calculate(currentNode, node);
}
@Override
public Double visit(Way way) {
log.error("Unable to process way");
return Double.MAX_VALUE;
}
@Override
public Double visit(Relation relation) {
log.error("Unable to process relation");
return Double.MAX_VALUE;
}
});
if (km < smallestDistance) {
smallestDistance = km;
if (km == 0) {
break;
}
}
}
distanceToPostorter.add(new Scored<Postort>(smallestDistance, segment.getPostnummer().getPostort()));
}
}
Collections.sort(distanceToPostorter, Scored.lowScoreFirstComparator);
boolean match = false;
if (distanceToPostorter.get(0).getScore() < matchDistanceKm) {
if (distanceToPostorter.size() == 1 || distanceToPostorter.get(1).getScore() > mismatchDistanceKm) {
match = true;
voronoiFactory.addCoordinate(distanceToPostorter.get(0).getObject(), node);
}
}
if (!match) {
System.currentTimeMillis();
}
System.currentTimeMillis();
}
}
}
}
// house number ways
{
Set<Postort> postorterChecked = new HashSet<Postort>();
parser.setRoot(new se.kodapan.osm.domain.root.Root());
parser.parse(overpass.execute("<osm-script>\n" +
"<query type=\"way\">\n" +
"<has-kv k=\"addr:housenumber\"/>\n" +
"<has-kv k=\"addr:street\"/>\n" +
"</query>\n" +
"<print/>\n" +
"</osm-script>"));
parser.parse(overpass.execute("<osm-script>\n" +
"<query type=\"way\">\n" +
"<has-kv k=\"addr:housenumber\"/>\n" +
"<has-kv k=\"addr:street\"/>\n" +
"</query>\n" +
"<recurse type=\"way-node\"/>\n" +
"<print/>\n" +
"</osm-script>"));
for (final Way way : parser.getRoot().getWays().values()) {
List<Scored<Postort>> distanceToPostorter = new ArrayList<Scored<Postort>>();
Collection<PostnummerSegment> segments = localPosten.getSegmentsByName().get(way.getTag("addr:street"));
if (segments != null) {
postorterChecked.clear();
for (final PostnummerSegment segment : segments) {
if (postorterChecked.add(segment.getPostnummer().getPostort())) {
if (segment.getPostnummer().getPostort().getReferencedObjects() == null || segment.getPostnummer().getPostort().getReferencedObjects().isEmpty()) {
log.error("Missing reference to postort " + segment.getPostnummer().getPostort());
distanceToPostorter.add(new Scored<Postort>(Double.MAX_VALUE, segment.getPostnummer().getPostort()));
} else {
double smallestDistance = Double.MAX_VALUE;
for (OsmObject osmObject : segment.getPostnummer().getPostort().getReferencedObjects()) {
double km = osmObject.accept(new OsmObjectVisitor<Double>() {
@Override
public Double visit(Node currentNode) {
return distance.calculate(currentNode, way.getNodes().get(0));
}
@Override
public Double visit(Way way) {
log.error("Unable to process way");
return Double.MAX_VALUE;
}
@Override
public Double visit(Relation relation) {
log.error("Unable to process relation");
return Double.MAX_VALUE;
}
});
if (km < smallestDistance) {
smallestDistance = km;
if (km == 0) {
break;
}
}
}
distanceToPostorter.add(new Scored<Postort>(smallestDistance, segment.getPostnummer().getPostort()));
}
}
Collections.sort(distanceToPostorter, Scored.lowScoreFirstComparator);
boolean match = false;
if (distanceToPostorter.get(0).getScore() < matchDistanceKm) {
if (distanceToPostorter.size() == 1 || distanceToPostorter.get(1).getScore() > mismatchDistanceKm) {
match = true;
for (Node node : way.getNodes()) {
voronoiFactory.addCoordinate(distanceToPostorter.get(0).getObject(), node);
}
}
}
if (!match) {
System.currentTimeMillis();
}
System.currentTimeMillis();
}
}
}
}
Map<Postort, List<Polygon>> voronoi = voronoiFactory.build();
serializeVoronoi(voronoiObjectFile, new JtsSerializer(geometryFactory), voronoi);
try {
OsmXmlWriter osmxml = new OsmXmlWriter(new OutputStreamWriter(new FileOutputStream(voronoiOsmFile), "utf8"));
AdjacentClassVoronoiClusterer.OsmRootFactory<Postort> factory = new AdjacentClassVoronoiClusterer.OsmRootFactory<Postort>() {
private String source = "Voronoi of points with ref:se:pts:postort set, tested against data available in Postnummersystemet. osm@kodapan.se";
@Override
public void setNode(Node node, Postort postort, List<Polygon> geometries, Polygon geometry, Coordinate coordinate) {
node.setTag("note", "Swedish postort border point. Might be inaccurate.");
node.setTag("source", source);
}
@Override
public void setWay(Way way, Postort postort, List<Polygon> geometries, Polygon geometry) {
way.setTag("note", "Swedish postort border. Might be inaccurate.");
way.setTag("source", source);
}
@Override
public void setClassTypeInstanceMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries) {
relation.setTag("note", postort.getIdentity() + "-group with " + voronoiFactory.getCoordinatesByClass().get(postort).size() + " points describing the Swedish postal town");
relation.setTag("source", source);
relation.setTag("ref:se:pts:postort", postort.getIdentity());
}
@Override
public void setGeometryMultiPolygon(Relation relation, Postort postort, List<Polygon> geometries, Polygon geometry) {
relation.setTag("note", "A complex polygon describing part of the Swedish postal town " + postort.getIdentity());
relation.setTag("source", source);
}
};
osmxml.write(factory.factory(voronoi));
osmxml.close();
} catch (IOException ioe) {
log.error("Could not write " + voronoiOsmFile.getAbsolutePath(), ioe);
}
}
private Map<Postort, List<Polygon>> deserializeVoronoi(File file, Postort borderland, JtsSerializer jts) throws IOException {
Map<Postort, List<Polygon>> classifications = new HashMap<Postort, List<Polygon>>();
ObjectInputStream objectInputStream = new ObjectInputStream(new FileInputStream(file));
while (objectInputStream.readBoolean()) {
String postortsIdentity = objectInputStream.readUTF();
if (postortsIdentity == null) {
System.currentTimeMillis();
}
Postort postort = localPosten.getPostortByIdentity().get(postortsIdentity);
if (postort == null) {
if (borderland.getIdentity().equals(postortsIdentity)) {
postort = borderland;
} else {
System.currentTimeMillis();
throw new NullPointerException();
}
}
List<Polygon> geometries = new ArrayList<Polygon>();
while (objectInputStream.readBoolean()) {
geometries.add(jts.readPolygon(objectInputStream));
}
classifications.put(postort, geometries);
}
objectInputStream.close();
return classifications;
}
private void serializeVoronoi(File file, JtsSerializer jts, Map<Postort, List<Polygon>> geometriesPerPostortRawOsmVoronoi) throws IOException {
ObjectOutputStream objectOutputStream = new ObjectOutputStream(new FileOutputStream(file));
for (Map.Entry<Postort, List<Polygon>> entry : geometriesPerPostortRawOsmVoronoi.entrySet()) {
objectOutputStream.writeBoolean(true);
if (entry.getKey() == null) {
System.currentTimeMillis();
}
objectOutputStream.writeUTF(entry.getKey().getIdentity());
for (Polygon geometry : entry.getValue()) {
objectOutputStream.writeBoolean(true);
Polygon polygon = (Polygon) geometry;
jts.writePolygon(polygon, objectOutputStream);
}
objectOutputStream.writeBoolean(false);
}
objectOutputStream.writeBoolean(false);
objectOutputStream.close();
}
}