/************************************************************************** OSMemory library for OSM data processing. Copyright (C) 2014 Aleś Bułojčyk <alex73mail@gmail.com> This is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This software is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. **************************************************************************/ package org.alex73.osmemory; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.channels.FileChannel; import java.nio.charset.Charset; import java.util.Arrays; /** * Driver for o5m read. Format described at the http://wiki.openstreetmap.org/wiki/O5m. */ public class O5MDriver { public static final Charset UTF8 = Charset.forName("UTF-8"); public static final byte MARK_BOF = (byte) 0xFF; public static final byte MARK_EOF = (byte) 0xFE; public static final byte MARK_RESET = (byte) 0xFF; public static final byte MARK_DATASET_NODE = (byte) 0x10; public static final byte MARK_DATASET_WAY = (byte) 0x11; public static final byte MARK_DATASET_RELATION = (byte) 0x12; public static final byte MARK_DATASET_BOUNDINGBOX = (byte) 0xDB; public static final byte MARK_DATASET_FILETIMESTAMP = (byte) 0xDC; public static final byte MARK_DATASET_HEADER = (byte) 0xE0; public static final byte MARK_DATASET_SYNC = (byte) 0xEE; public static final byte MARK_DATASET_JUMP = (byte) 0xEF; final O5MReader handler; private ByteBuffer buffer; private DeltaCoder deltaId = new DeltaCoder(); private DeltaCoder deltaTimestamp = new DeltaCoder(); private DeltaCoder deltaChangeset = new DeltaCoder(); private DeltaCoderInt deltaLongitude = new DeltaCoderInt(); private DeltaCoderInt deltaLatitude = new DeltaCoderInt(); private DeltaCoder deltaReferenceWayNode = new DeltaCoder(); private DeltaCoder deltaReferenceRelationMemberNode = new DeltaCoder(); private DeltaCoder deltaReferenceRelationMemberWay = new DeltaCoder(); private DeltaCoder deltaReferenceRelationMemberRelation = new DeltaCoder(); private int[] stringPairPositions = new int[15001]; private int[] stringPairFirstSizes = new int[15001]; private int[] stringPairSecondSizes = new int[15001]; int stringPairPos; private int datasetEndPos; private int objectTagsCount; private int[] objectTagKeyPositions = new int[1024]; private int[] objectTagKeySizes = new int[1024]; private int[] objectTagValuePositions = new int[1024]; private int[] objectTagValueSizes = new int[1024]; private long[] nodes = new long[8192]; private byte[] memberTypes = new byte[8192]; private int[] memberRolePositions = new int[8192]; private int[] memberRoleSizes = new int[8192]; private long currentId; private long currentTimestamp; private int currentVersion; private long currentChangeset; private String currentUser; private void resetDeltas() { deltaId.value = 0; deltaTimestamp.value = 0; deltaChangeset.value = 0; deltaLongitude.value = 0; deltaLatitude.value = 0; deltaReferenceWayNode.value = 0; deltaReferenceRelationMemberNode.value = 0; deltaReferenceRelationMemberWay.value = 0; deltaReferenceRelationMemberRelation.value = 0; } public O5MDriver(O5MReader handler) { this.handler = handler; } public void read(File file) throws Exception { handler.fileTimestamp(file.lastModified()); try (RandomAccessFile aFile = new RandomAccessFile(file, "r")) { try (FileChannel inChannel = aFile.getChannel()) { buffer = inChannel.map(FileChannel.MapMode.READ_ONLY, 0, file.length()); buffer.order(ByteOrder.LITTLE_ENDIAN); if (buffer.get() != MARK_BOF) { throw new IOException("This is not a .o5m file"); } while (buffer.remaining() > 0) { byte datasetType = buffer.get(); if (datasetType == MARK_RESET) { resetDeltas(); continue; } if (datasetType == MARK_EOF && buffer.remaining() == 0) { break; } long datasetLength = readUnsignedNumberAbsolute(); datasetEndPos = (int) (buffer.position() + datasetLength); switch (datasetType) { case MARK_DATASET_HEADER: // should follow the first 0xff in the file; contents: 0x04 0x6f 0x35 0x6d 0x32 // ("o5m2"), // or 0x04 0x6f 0x35 0x63 0x32 ("o5c2") for .o5m change files break; case MARK_DATASET_FILETIMESTAMP: readTimestamp(); break; case MARK_DATASET_BOUNDINGBOX: break; case MARK_DATASET_NODE: readNode(); break; case MARK_DATASET_WAY: readWay(); break; case MARK_DATASET_RELATION: readRelation(); break; default: throw new RuntimeException("Unknown dataset : " + Integer.toHexString(datasetType)); } objectTagsCount = 0; buffer.position(datasetEndPos); } } } } void readTimestamp() { handler.fileTimestamp(readSignedNumber(new DeltaCoder()) * 1000); } void readNode() { currentId = readSignedNumber(deltaId); if (buffer.position() >= datasetEndPos) { return; } readVersion(); if (buffer.position() >= datasetEndPos) { return; } int longitude = readSignedNumberInt(deltaLongitude); int latitude = readSignedNumberInt(deltaLatitude); while (buffer.position() < datasetEndPos) { readObjectTag(); } handler.createNode(this, currentId, latitude, longitude, currentUser); } void readWay() { currentId = readSignedNumber(deltaId); if (buffer.position() >= datasetEndPos) { return; } readVersion(); if (buffer.position() >= datasetEndPos) { return; } int refSectionLength = (int) readUnsignedNumberAbsolute(); int refSectionEnd = buffer.position() + refSectionLength; int c = 0; while (buffer.position() < refSectionEnd) { nodes[c] = readSignedNumber(deltaReferenceWayNode); c++; } while (buffer.position() < datasetEndPos) { readObjectTag(); } handler.createWay(this, currentId, Arrays.copyOf(nodes, c), currentUser); } void readRelation() { currentId = readSignedNumber(deltaId); if (buffer.position() >= datasetEndPos) { return; } readVersion(); if (buffer.position() >= datasetEndPos) { return; } int refSectionLength = (int) readUnsignedNumberAbsolute(); int refSectionEnd = buffer.position() + refSectionLength; int c = 0; while (buffer.position() < refSectionEnd) { long idval = readUnsignedNumberAbsolute(); memberTypes[c] = readMemberInfo(c); switch (memberTypes[c]) { case IOsmObject.TYPE_NODE: nodes[c] = readSignedNumber(idval, deltaReferenceRelationMemberNode); break; case IOsmObject.TYPE_WAY: nodes[c] = readSignedNumber(idval, deltaReferenceRelationMemberWay); break; case IOsmObject.TYPE_RELATION: nodes[c] = readSignedNumber(idval, deltaReferenceRelationMemberRelation); break; } c++; if (c >= memberTypes.length) { throw new RuntimeException("Too many members in relation"); } } while (buffer.position() < datasetEndPos) { readObjectTag(); } handler.createRelation(this, currentId, Arrays.copyOf(nodes, c), Arrays.copyOf(memberTypes, c), currentUser); } void readVersion() { currentVersion = 0; currentTimestamp = 0; currentChangeset = 0; currentVersion = (int) readUnsignedNumberAbsolute(); if (currentVersion == 0) { return; } currentTimestamp = readSignedNumber(deltaTimestamp); if (currentTimestamp == 0) { return; } currentChangeset = readSignedNumber(deltaChangeset); // read uid user pair long v = readUnsignedNumberAbsolute(); int pairPos; if (v != 0) { // refer to string pair pairPos = (int) (stringPairPos - v); if (pairPos < 0) { pairPos += stringPairPositions.length; } } else { pairPos = stringPairPos; storeStringPair(stringPairPos); if (stringPairFirstSizes[stringPairPos] + stringPairSecondSizes[stringPairPos] <= 250) { // store for future stringPairPos++; if (stringPairPos >= stringPairPositions.length) { stringPairPos -= stringPairPositions.length; } } } currentUser = getString(stringPairPositions[pairPos] + stringPairFirstSizes[pairPos] + 1, stringPairSecondSizes[pairPos]); } /** * To store numbers of different lengths we abandon bit 7 (most significant bit) of every byte and use * this bit as a length indicator. This indicator – when set to 1 – tells us that the next byte belongs to * the same number. The first byte of such a long number contains the least significant 7 bits, the last * byte the most significant 7 bits. */ long readUnsignedNumberAbsolute() { long value = 0; long b = 0x80; for (int i = 0; (b & 0x80) != 0; i += 7) { b = buffer.get(); long part = (b & 0x7F) << i; value |= part; } return value; } /** * If a number is stored as "signed", we will need 1 bit for the sign. For this purpose, the least * significant bit of the least significant byte is taken as sign bit. 0 means positive, 1 means negative. * We do not need the number -0, of course, so we can shift the range of negative numbers by one. */ long readSignedNumber(DeltaCoder delta) { return readSignedNumber(readUnsignedNumberAbsolute(), delta); } long readSignedNumber(long value, DeltaCoder delta) { if ((value & 1) != 0) { // negative value = -(value >> 1) - 1; } else { // positive value = (value >> 1); } long result = delta.value + value; delta.value = result; return result; } int readSignedNumberInt(DeltaCoderInt delta) { long value = readUnsignedNumberAbsolute(); if ((value & 1) != 0) { // negative value = -(value >> 1) - 1; } else { // positive value = (value >> 1); } int result = (int) (delta.value + value); delta.value = result; return result; } byte readMemberInfo(int i) { long v = readUnsignedNumberAbsolute(); int pairPos; if (v != 0) { // refer to string pair pairPos = (int) (stringPairPos - v); if (pairPos < 0) { pairPos += stringPairPositions.length; } } else { pairPos = stringPairPos; storeStringOnes(stringPairPos); if (stringPairFirstSizes[stringPairPos] + stringPairSecondSizes[stringPairPos] <= 250) { // store for future stringPairPos++; if (stringPairPos >= stringPairPositions.length) { stringPairPos -= stringPairPositions.length; } } } memberRolePositions[i] = stringPairPositions[pairPos] + 1; memberRoleSizes[i] = stringPairFirstSizes[pairPos] - 1; switch (buffer.get(stringPairPositions[pairPos])) { case '0': return OsmNode.TYPE_NODE; case '1': return OsmNode.TYPE_WAY; case '2': return OsmNode.TYPE_RELATION; default: throw new RuntimeException("Unknown member type"); } } void readObjectTag() { long v = readUnsignedNumberAbsolute(); int pairPos; if (v != 0) { // refer to string pair pairPos = (int) (stringPairPos - v); if (pairPos < 0) { pairPos += stringPairPositions.length; } } else { pairPos = stringPairPos; storeStringPair(stringPairPos); if (stringPairFirstSizes[stringPairPos] + stringPairSecondSizes[stringPairPos] <= 250) { // store for future stringPairPos++; if (stringPairPos >= stringPairPositions.length) { stringPairPos -= stringPairPositions.length; } } } objectTagKeyPositions[objectTagsCount] = stringPairPositions[pairPos]; objectTagKeySizes[objectTagsCount] = stringPairFirstSizes[pairPos]; objectTagValuePositions[objectTagsCount] = stringPairPositions[pairPos] + stringPairFirstSizes[pairPos] + 1; objectTagValueSizes[objectTagsCount] = stringPairSecondSizes[pairPos]; objectTagsCount++; if (objectTagsCount >= objectTagKeyPositions.length) { throw new RuntimeException("Too many tags ib object"); } } void storeStringPair(int pos) { stringPairPositions[pos] = buffer.position(); int p1 = 0; while (buffer.get() != 0) { p1++; } int p2 = 0; while (buffer.get() != 0) { p2++; } stringPairFirstSizes[pos] = p1; stringPairSecondSizes[pos] = p2; } void storeStringOnes(int pos) { stringPairPositions[pos] = buffer.position(); int p1 = 0; while (buffer.get() != 0) { p1++; } stringPairFirstSizes[pos] = p1; stringPairSecondSizes[pos] = 0; } void skip(long count) { for (int i = 0; i < count; i++) { buffer.get(); } } int getObjectTagsCount() { return objectTagsCount; } String getObjectTagKeyString(int pos) { return getString(objectTagKeyPositions[pos], objectTagKeySizes[pos]); } byte[] getObjectTagValueBytes(int pos) { return getBytes(objectTagValuePositions[pos], objectTagValueSizes[pos]); } String getMemberRoleString(int pos) { return getString(memberRolePositions[pos], memberRoleSizes[pos]); } public long getCurrentChangeset() { return currentChangeset; } String getString(int pos, int len) { int p = buffer.position(); buffer.position(pos); byte[] result = new byte[len]; buffer.get(result); buffer.position(p); return new String(result, UTF8); } byte[] getBytes(int pos, int len) { int p = buffer.position(); buffer.position(pos); byte[] result = new byte[len]; buffer.get(result); buffer.position(p); return result; } static class DeltaCoder { long value; } static class DeltaCoderInt { int value; } }