package se.kodapan.osm.sweden.ext.se.posten.postnummer;
import org.apache.commons.codec.binary.Hex;
import org.wikipedia.WikiBot;
import se.kodapan.osm.parser.xml.instantiated.InstantiatedOsmXmlParser;
import se.kodapan.osm.domain.*;
import se.kodapan.osm.domain.root.ContainsTagKeyValueFilter;
import se.kodapan.osm.domain.root.Root;
import se.kodapan.osm.util.distance.ArcDistance;
import se.kodapan.osm.util.distance.OsmObjectCentroidDistance;
import se.kodapan.osm.util.distance.OsmObjectDistance;
import se.kodapan.osm.services.nominatim.Nominatim;
import se.kodapan.osm.services.nominatim.NominatimJsonResponseParser;
import se.kodapan.osm.services.nominatim.NominatimQueryBuilder;
import se.kodapan.osm.services.overpass.Overpass;
import se.kodapan.osm.services.overpass.OverpassUtils;
import se.kodapan.osm.sweden.ext.wikipedia.WikipediaTools;
import java.io.File;
import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.security.MessageDigest;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* This robot can take a whole day to run when not using cached data from PostenPostnummerService.
*
* @author kalle
* @since 2013-07-29 18:31
*/
public class PostnummersystemetRobot {
private static OsmObjectVisitor<String> osmWikiLinkVisitor = new OsmObjectVisitor<String>() {
@Override
public String visit(Node node) {
return "{{Node|" + node.getId() + "}}";
}
@Override
public String visit(Way way) {
return "{{Way|" + way.getId() + "}}";
}
@Override
public String visit(Relation relation) {
return "{{Relation|" + relation.getId() + "}}";
}
};
public static void main(String[] args) throws Exception {
/** This robot can take a whole day to run when not using cached data from PostenPostnummerService. */
boolean useCache = true;
File cachePath = new File("target/" + PostnummersystemetRobot.class.getName());
String userAgent = "Project OSM-Sweden robot, http://wiki.openstreetmap.org/wiki/User:Osm.kodapan.se/bots/Postnummersystemet";
WikiBot wiki = new WikiBot("wiki.openstreetmap.org");
wiki.setBotName("Postnummersystemet");
wiki.setUsingCompressedRequests(false);
wiki.login("osm.kodapan.se", System.getProperty("wiki.openstreetmap.org.password"));
try {
wiki.addToBotWikiLog("Started");
Root root = new Root();
InstantiatedOsmXmlParser parser = new InstantiatedOsmXmlParser();
parser.setRoot(root);
NominatimJsonResponseParser jsonResponseParser = new NominatimJsonResponseParser();
jsonResponseParser.setRoot(root);
Nominatim nominatim = new Nominatim();
nominatim.setUserAgent(userAgent);
nominatim.open();
OsmObjectDistance distance = new OsmObjectCentroidDistance(new ArcDistance());
try {
Overpass overpass;
// if (useCache) {
// overpass = new FileSystemCachedOverpass(new File(cachePath, "overpass"));
// } else {
overpass = new Overpass();
// }
overpass.setUserAgent(userAgent);
overpass.open();
OverpassUtils overpassUtils = new OverpassUtils(overpass);
// load all known postorter
parser.parse(new StringReader(
overpass.execute("<osm-script>\n" +
" <query type=\"node\">\n" +
" <has-kv k=\"ref:se:pts:postort\" />\n" +
" </query>\n" +
" <print/>\n" +
"</osm-script>")));
Set<OsmObject> objectsInOsmWithPostortTag = root.gatherAllOsmObjects();
try {
PostenPostnummerService posten;
CachedPostenPostnummerService cachedPosten = new CachedPostenPostnummerService(useCache);
cachedPosten.setPath(new File(cachePath, "posten"));
posten = cachedPosten;
posten.setUserAgent(userAgent);
posten.open();
try {
Map<String, Set<String>> postalCodeByPostorter;
postalCodeByPostorter = posten.gatherAllPostnummerByPostorter(10000, 99999);
Set<String> postalCodes = new HashSet<String>(10000);
for (Map.Entry<String, Set<String>> entry : postalCodeByPostorter.entrySet()) {
postalCodes.addAll(entry.getValue());
}
Map<String, PostenPostnummerServiceResponse> allServiceRecordsByPostalCode = new HashMap<String, PostenPostnummerServiceResponse>();
for (String postalCode : postalCodes) {
allServiceRecordsByPostalCode.put(postalCode, posten.resolveAllServiceRecordsInPostnummer(postalCode));
}
// all data received.
// now bounce it around in maps and sets
// so get formatted the way we want it.
List<Map.Entry<String, Set<String>>> postorterAndPostalCodesOrderedByPostalCodes = new ArrayList<Map.Entry<String, Set<String>>>(postalCodeByPostorter.entrySet());
Collections.sort(postorterAndPostalCodesOrderedByPostalCodes, new Comparator<Map.Entry<String, Set<String>>>() {
@Override
public int compare(Map.Entry<String, Set<String>> o1, Map.Entry<String, Set<String>> o2) {
return greatestPostalCode(o1.getValue()).compareTo(greatestPostalCode(o2.getValue()));
}
public String greatestPostalCode(Set<String> set) {
String greatest = "";
for (String postalCode : set) {
if (greatest.compareTo(postalCode) < 0) {
greatest = postalCode;
}
}
return greatest;
}
});
List<Object[]> tableAll = new ArrayList<Object[]>(3000);
List<Object[]> tableProblematic = new ArrayList<Object[]>(3000);
Object[] headers = new Object[]{
"tag=ref:se:pts:postort",
"Antal postnummer",
// "Postnummer",
"Antal gator",
"Objekt i OSM med tag satt",
"Kommentar"};
tableAll.add(headers);
tableProblematic.add(headers);
for (Map.Entry<String, Set<String>> postortAndPostalCodes : postorterAndPostalCodesOrderedByPostalCodes) {
boolean problematicRow = false;
String postort = postortAndPostalCodes.getKey();
List<PostenPostnummerServiceRecord> postortServiceRecords = new ArrayList<PostenPostnummerServiceRecord>();
for (String postalCode : postortAndPostalCodes.getValue()) {
postortServiceRecords.addAll(allServiceRecordsByPostalCode.get(postalCode).getRecords());
}
Collections.sort(postortServiceRecords, new Comparator<PostenPostnummerServiceRecord>() {
@Override
public int compare(PostenPostnummerServiceRecord o1, PostenPostnummerServiceRecord o2) {
int ret = o1.getPostnummer().compareTo(o2.getPostnummer());
if (ret == 0) {
ret = o1.getGatunamn().compareTo(o2.getGatunamn());
if (ret == 0) {
ret = o1.getStarthusnummer() - o2.getStarthusnummer();
}
}
return ret;
}
});
System.currentTimeMillis();
List<String> postalCodesOrdered = new ArrayList<String>(postortAndPostalCodes.getValue());
Collections.sort(postalCodesOrdered);
StringBuilder postalCodesBuilder = new StringBuilder(4096);
for (String postalCode : postalCodesOrdered) {
postalCodesBuilder.append(postalCode).append("<br/>");
}
List<String> streets = new ArrayList<String>(250);
for (PostenPostnummerServiceRecord record : postortServiceRecords) {
if (!streets.contains(record.getGatunamn())) {
streets.add(record.getGatunamn());
}
}
Collections.sort(streets);
StringBuilder kommentar = new StringBuilder(1024);
StringBuilder osmObjectsWithTagLinks = new StringBuilder(1024);
Collection<OsmObject> osmObjectsWithTagKeyValue = root.filter(objectsInOsmWithPostortTag, new ContainsTagKeyValueFilter("ref:se:pts:postort", postort));
if (osmObjectsWithTagKeyValue.isEmpty()) {
problematicRow = true;
NominatimQueryBuilder nominatimQueryBuilder = new NominatimQueryBuilder()
.setQuery(new StringBuilder()
.append(postort)
.append(", sverige")
.toString())
.addCountryCode("se")
.setLimit(10);
String nominatimQueryWikiLink = "[" + nominatimQueryBuilder.build() + " Nominatim]";
nominatimQueryBuilder.setFormat("json");
String nominatimJsonResponse = nominatim.search(nominatimQueryBuilder.build());
try {
List<NominatimJsonResponseParser.Result> results = jsonResponseParser.parse(nominatimJsonResponse);
overpassUtils.loadAllObjects(root);
// List<OsmObject> osmObjects = new ArrayList<OsmObject>(results.size());
// for (NominatimJsonResponseParser.Result result : results) {
// osmObjects.add(result.getObject());
// }
// root.filter(osmObjects, new ContainsTagKeyFilter("place"));
if (results.isEmpty()) {
kommentar.append("Ingen träff på postortsnamn i " + nominatimQueryWikiLink);
} else {
if (results.size() == 1) {
kommentar.append(String.valueOf("1 träff på postortsnamn i " + nominatimQueryWikiLink + "<br/><br/>"));
NominatimJsonResponseParser.Result result = results.iterator().next();
printNominatimResult(osmWikiLinkVisitor, kommentar, result);
} else {
kommentar.append(String.valueOf(results.size())).append(" träffar på postortsnamn i " + nominatimQueryWikiLink);
double maximumKilometers = -1;
List<OsmObject> osmObjects = new ArrayList<OsmObject>(results.size());
for (NominatimJsonResponseParser.Result object : results) {
osmObjects.add(object.getObject());
}
for (OsmObject object1 : osmObjects) {
for (OsmObject object2 : osmObjects) {
double kilometers = distance.calculate(object1, object2);
if (maximumKilometers < kilometers) {
maximumKilometers = kilometers;
}
}
}
if (maximumKilometers == -1) {
System.currentTimeMillis();
}
if (maximumKilometers == 0) {
kommentar.append(" som delar samma position");
} else if (maximumKilometers < 1) {
kommentar.append(" inom ").append(String.valueOf((int) (maximumKilometers * 1000))).append(" meter");
} else {
kommentar.append(" inom ").append(String.valueOf((int) maximumKilometers)).append(" kilometer");
}
kommentar.append(":<br/><br/>");
Iterator<NominatimJsonResponseParser.Result> resultIterator = results.iterator();
while (resultIterator.hasNext()) {
NominatimJsonResponseParser.Result result = resultIterator.next();
printNominatimResult(osmWikiLinkVisitor, kommentar, result);
if (resultIterator.hasNext()) {
kommentar.append("<br/><br/>");
}
}
}
}
System.currentTimeMillis();
} catch (Exception e) {
e.printStackTrace();
System.currentTimeMillis();
}
} else {
// distance if many
if (osmObjectsWithTagKeyValue.size() > 1) {
problematicRow = true;
kommentar.append(String.valueOf(osmObjectsWithTagKeyValue.size())).append(" träffar med ref:se:pts:postort satt i OSM ");
double maximumKilometers = -1;
for (OsmObject object1 : osmObjectsWithTagKeyValue) {
for (OsmObject object2 : osmObjectsWithTagKeyValue) {
double kilometers = distance.calculate(object1, object2);
if (maximumKilometers < kilometers) {
maximumKilometers = kilometers;
}
}
}
if (maximumKilometers == 0) {
kommentar.append(" som delar samma position");
} else if (maximumKilometers < 1) {
kommentar.append("inom ").append(String.valueOf((int) (maximumKilometers * 1000))).append(" meter");
} else {
kommentar.append("inom ").append(String.valueOf((int) maximumKilometers)).append(" kilometer");
}
kommentar.append(":<br/>");
}
for (Iterator<OsmObject> iterator = osmObjectsWithTagKeyValue.iterator(); iterator.hasNext(); ) {
OsmObject object = iterator.next();
String name = object.getTag("name");
osmObjectsWithTagLinks.append("name=").append(name).append("<br/>");
osmObjectsWithTagLinks.append(object.accept(osmWikiLinkVisitor));
if (iterator.hasNext()) {
osmObjectsWithTagLinks.append("<br/><br/>");
}
}
}
// Object[] headers = new Object[]{"tag=ref:se:pts:postort", "Antal postnummer", "Postnummer", "Antal gator", "Objekt i OSM med tag satt", "Kommentar"};
Object[] row = new Object[]{
postort,
postalCodeByPostorter.get(postort).size(),
// postalCodesBuilder.toString(),
streets.size(),
osmObjectsWithTagLinks.toString(),
kommentar.toString()
};
tableAll.add(row);
if (problematicRow) {
tableProblematic.add(row);
}
}
System.currentTimeMillis();
StringWriter sw = new StringWriter(49152);
PrintWriter out = new PrintWriter(sw);
out.println("===Postorter som behöver arbete===");
WikipediaTools.renderTable(tableProblematic, out);
out.println("===Alla postorter===");
WikipediaTools.renderTable(tableAll, out);
out.flush();
String tables = sw.toString();
MessageDigest md = MessageDigest.getInstance("SHA-1");
String checksum = Hex.encodeHexString(md.digest(tables.getBytes()));
sw = new StringWriter(49152);
out = new PrintWriter(sw);
out.println("===Om innehållet på denna denna sidan===");
out.println("Informationen på denna sida genereras av roboten [[" + wiki.getBotHomePageTitle() + "]].<br/>");
out.println("Senaste uppdateringen skedde " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "<br/>");
out.println("Datan är en samkörning av alla objekt i OSM med ref:se:pts:postort=*, [http://www.posten.se/soktjanst/postnummersok Postens postnummersök] och Nominatim.<br/>");
out.println("");
out.print(tables);
out.println("===Kontrollsumma===");
out.println("Sista raden på denna sida är en HEX-kodad SHA1-kontrollsumma av sidans genererade innehåll.<br/>");
out.print(checksum);
out.flush();
String pageText = sw.toString();
String pageName = "WikiProject_Sweden/Postorter";
String currentRevision = wiki.getPageText(pageName);
if (currentRevision != null && currentRevision.trim().endsWith(checksum.trim())) {
wiki.addToBotWikiLog("No changes detected, leaving current page untouched.");
} else {
wiki.addToBotWikiLog("Changes detected, updating.");
wiki.edit(pageName, pageText, "Checksum does not match.");
}
} finally {
posten.close();
}
} finally {
overpass.close();
}
} finally {
nominatim.close();
}
wiki.addToBotWikiLog("Robot shutting down after successfully running the job.");
} catch (Exception e) {
e.printStackTrace();
wiki.addToBotWikiLog("Error! Robot shutting down due to uncaught unhandled exception, no changes!", e);
} finally {
wiki.logout();
}
}
private static void printNominatimResult(OsmObjectVisitor<String> osmWikiLinkVisitor, StringBuilder kommentar, NominatimJsonResponseParser.Result result) {
kommentar.append(prefix(60, result.getJsonObject().get("display_name").toString())).append("<br/>");
String place = result.getObject().getTag("place");
if (place != null) {
kommentar.append("place=").append(place).append("<br/>");
}
kommentar.append(result.getObject().accept(osmWikiLinkVisitor));
}
private static String prefix(int length, String input) {
return input.substring(0, input.length() < length ? input.length() : length) + (input.length() > length ? "..." : "");
}
}