commit 8d208064a7c97b61ccdbda45dd774cb4ade2cc98 parent 426c5db37f16561be6aa33019ff0a522b14f74a6 Author: Friedel Schön <derfriedmundschoen@gmail.com> Date: Fri, 8 Mar 2024 15:45:04 +0100 . Diffstat:
26 files changed, 2316 insertions(+), 991 deletions(-)
diff --git a/.vscode/settings.json b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "java.configuration.updateBuildConfiguration": "interactive" +} +\ No newline at end of file diff --git a/classes.sh b/classes.sh @@ -0,0 +1,7 @@ +find -name '*.java' | # all java files in this directory + grep -v 'src/test' | # skip all test files + sed -E 's|^\./[a-z-]+/src/main/java/||;s/\.java$//;s|/|.|g;s/(.*)+\./\1 /' | + # remove path-prefix remove end / -> . split package/class + sort | # sort by package + awk '{ if(NR > 1 && $1 != current) print ""; if($1 != current) print $1 ":"; current=$1; print "- " $2 }' | # print by package + tee classes.txt # write to classes.txt diff --git a/classes.txt b/classes.txt @@ -0,0 +1,67 @@ +osm.common: +- ArrayIterable +- ConsumingIterator +- Container +- ContainerCollector +- Edge +- RandomAccessFileInputStream +- TagMap + +osm.geo: +- Point + +osm.message: +- AbstractEntity +- Blob +- BlobHeader +- DenseNodes +- Entity +- HeaderBBox +- HeaderBlock +- Node +- PrimitiveBlock +- PrimitiveGroup +- Relation +- StringTable +- Way + +osm.planner: +- BasePlanner +- Planner +- PlannerFunction +- TargetIterable + +osm.protobuf: +- BlobSpliterator +- Graph + +osm.routing: +- RoutingEdge +- RoutingGraph +- RoutingNode +- RoutingStrategy + +osm.routing.entity: +- Interval +- Passenger + +osm.routing.strategy: +- BestFirstRouter +- DijkstraRouter +- EuclidianRouter + +persolijn: +- App +- WinschotenTest + +protobuf: +- Message +- MessageIterator +- ProtobufReader +- WireType + +protobuf.exception: +- InputException +- OverflowException +- UnexpectedTagException +- WireTypeException diff --git a/osm-protobuf/src/main/java/osm/common/ConsumingIterator.java b/osm-protobuf/src/main/java/osm/common/ConsumingIterator.java @@ -17,6 +17,12 @@ public class ConsumingIterator<T> implements Iterator<T> { this.consumer = consumer; } + public ConsumingIterator(Iterator<T> original, Predicate<T> consumeWhere) { + this.original = original; + this.consumeWhere = consumeWhere; + this.consumer = null; + } + @Override public boolean hasNext() { if (next != null) @@ -27,7 +33,8 @@ public class ConsumingIterator<T> implements Iterator<T> { if (!consumeWhere.test(next)) return true; - consumer.accept(next); + if (consumer != null) + consumer.accept(next); } return false; } diff --git a/osm-protobuf/src/main/java/osm/common/TagMap.java b/osm-protobuf/src/main/java/osm/common/TagMap.java @@ -3,6 +3,7 @@ package osm.common; import java.util.AbstractMap; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; @@ -139,22 +140,17 @@ public class TagMap implements Map<String, String> { @Override public String toString() { - StringBuilder str = new StringBuilder("{ "); - boolean first = true; - - for (Entry<String, String> entry : entrySet()) { - if (!first) { - str.append(", "); - } - first = false; - str.append(entry.getKey()); - str.append("="); - str.append(entry.getValue()); - } - - str.append(" }"); - - return str.toString(); + return entrySet().stream() + .collect(() -> new StringBuilder("{"), (str, entry) -> { + if (str.length() > 1) + str.append(", "); + + str.append(entry.getKey()); + str.append("="); + str.append(entry.getValue()); + }, StringBuilder::append) + .append('}') + .toString(); } @Override diff --git a/osm-protobuf/src/main/java/osm/message/AbstractEntity.java b/osm-protobuf/src/main/java/osm/message/AbstractEntity.java @@ -83,17 +83,19 @@ abstract class AbstractEntity<T> implements Message<T>, Entity { @Override public T parse(Iterator<ProtobufReader> tags) { return parseRemaining( - new ConsumingIterator<>(tags, - message -> message.tag() == 1 || message.tag() == 2 || message.tag() == 3, - message -> { - switch (message.tag()) { - case 1 -> id = message.varint64(); - case 2 -> message.packed(message::varint32, block.stringtable::get) - .forEachRemaining(keys::add); - case 3 -> message.packed(message::varint32, block.stringtable::get) - .forEachRemaining(values::add); - } - })); + new ConsumingIterator<>(tags, message -> { + switch (message.tag()) { + case 1 -> id = message.varint64(); + case 2 -> message.packed(message::varint32, block.stringtable::get) + .forEachRemaining(keys::add); + case 3 -> message.packed(message::varint32, block.stringtable::get) + .forEachRemaining(values::add); + default -> { + return false; + } + } + return true; + })); } /** diff --git a/osm-protobuf/src/main/java/osm/message/Blob.java b/osm-protobuf/src/main/java/osm/message/Blob.java @@ -5,6 +5,7 @@ import java.util.zip.Inflater; import protobuf.Message; import protobuf.ProtobufReader; +import java.io.UnsupportedEncodingException; import java.util.Iterator; import java.util.List; import java.util.zip.DataFormatException; diff --git a/osm-protobuf/src/main/java/osm/message/BlobHeader.java b/osm-protobuf/src/main/java/osm/message/BlobHeader.java @@ -15,7 +15,6 @@ public class BlobHeader implements Message<BlobHeader> { @Override public BlobHeader parse(Iterator<ProtobufReader> iter) { - while (iter.hasNext()) { ProtobufReader message = iter.next(); switch (message.tag()) { diff --git a/osm-protobuf/src/main/java/osm/message/DenseNodes.java b/osm-protobuf/src/main/java/osm/message/DenseNodes.java @@ -38,6 +38,7 @@ public class DenseNodes implements Message<List<Node>> { while (iter.hasNext()) { ProtobufReader stream = iter.next(); + switch (stream.tag()) { case 1: Iterator<Long> ids = stream.packed(stream::svarint64, 0l, Long::sum); diff --git a/osm-protobuf/src/main/java/osm/message/Relation.java b/osm-protobuf/src/main/java/osm/message/Relation.java @@ -1,6 +1,9 @@ package osm.message; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; +import java.util.function.Supplier; import protobuf.ProtobufReader; @@ -17,19 +20,86 @@ import protobuf.ProtobufReader; // repeated sint64 memids = 9 [packed = true]; // DELTA encoded // repeated MemberType types = 10 [packed = true]; public class Relation extends AbstractEntity<Relation> { + public static enum RelationMemberType { + NODE, WAY, RELATION + } + + public static class RelationMember { + private RelationMemberType type; + private long id; + private String role; + + public RelationMemberType getType() { + return type; + } + + public long getID() { + return id; + } + + public String getRole() { + return role; + } + } + + private List<RelationMember> members = new ArrayList<>(); + public Relation(PrimitiveBlock block) { super(block); } + private <T> void expandList(List<T> list, int index, Supplier<T> with) { + while (list.size() <= index) + list.add(with.get()); + } + @Override public Relation parseRemaining(Iterator<ProtobufReader> tags) { + int indexRole = 0, + indexID = 0, + indexType = 0; + while (tags.hasNext()) { ProtobufReader message = tags.next(); switch (message.tag()) { - case 4, 8, 9, 10 -> message.skip(); - default -> message.throwUnexpected(); + case 4: + message.skip(); + break; + case 8: + Iterator<Integer> roles = message.packed(message::varint32); + while (roles.hasNext()) { + expandList(members, indexRole, RelationMember::new); + + members.get(indexRole).role = block.stringtable.get(roles.next()); + indexRole++; + } + break; + case 9: + Iterator<Long> ids = message.packed(message::svarint64, 0L, Long::sum); + while (ids.hasNext()) { + expandList(members, indexID, RelationMember::new); + + members.get(indexID).id = ids.next(); + indexID++; + } + break; + case 10: + Iterator<Integer> types = message.packed(message::varint32); + while (types.hasNext()) { + expandList(members, indexType, RelationMember::new); + + members.get(indexType).type = RelationMemberType.values()[types.next()]; + indexType++; + } + break; + default: + message.throwUnexpected(); } } return this; } + + public List<RelationMember> getMembers() { + return members; + } } \ No newline at end of file diff --git a/osm-protobuf/src/main/java/osm/message/Way.java b/osm-protobuf/src/main/java/osm/message/Way.java @@ -85,8 +85,9 @@ public class Way extends AbstractEntity<Way> { protected double getSpeed(Direction direction) { String speed = "60"; + String defaultSpeed; - speed = WAY_DEFAULT_SPEED.getOrDefault(tags.get("highway"), speed); + defaultSpeed = speed = WAY_DEFAULT_SPEED.getOrDefault(tags.get("highway"), speed); speed = tags.getOrDefault("maxspeed", speed); if (direction == Direction.BACKWARDS) @@ -95,7 +96,11 @@ public class Way extends AbstractEntity<Way> { if (speed.equals("none")) speed = WAY_UNLIMITED_SPEED; - return Double.parseDouble(speed) / 3.6; + try { + return Double.parseDouble(speed) / 3.6; + } catch (NumberFormatException exc) { + return Double.parseDouble(defaultSpeed) / 3.6; + } } @Override diff --git a/osm-protobuf/src/test/java/osm/protobuf/NodeTest.java b/osm-protobuf/src/test/java/osm/protobuf/NodeTest.java @@ -0,0 +1,343 @@ +package osm.protobuf; + +import java.io.File; +import java.io.RandomAccessFile; +import java.net.URI; +import java.util.Iterator; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import osm.message.Entity; +import osm.message.Node; + +/** + * Demonstrates how to read a file. Reads sample.pbf from the resources folder + * and prints details about it to the standard output. + * + * @author Michael Tandy + */ +public class NodeTest { + public static record ExpectedNode(long id, double latitude, double longitude) { + } + + @Test + public void test() throws Exception { + ExpectedNode[] expected = new ExpectedNode[] { + new ExpectedNode(653970877, 51.763603, -0.228757), + new ExpectedNode(647105170, 51.763591, -0.234465), + new ExpectedNode(672663476, 51.765749, -0.229070), + new ExpectedNode(241806356, 51.768945, -0.232662), + new ExpectedNode(692945017, 51.766185, -0.230069), + new ExpectedNode(1709246734, 51.766433, -0.230854), + new ExpectedNode(175685506, 51.765169, -0.229374), + new ExpectedNode(647105129, 51.769327, -0.218457), + new ExpectedNode(647105160, 51.768192, -0.231686), + new ExpectedNode(672663473, 51.765530, -0.229187), + new ExpectedNode(647105141, 51.773204, -0.222598), + new ExpectedNode(25365926, 51.766340, -0.233556), + new ExpectedNode(1685167296, 51.766924, -0.234783), + new ExpectedNode(677439943, 51.763178, -0.230230), + new ExpectedNode(1701110757, 51.766400, -0.228489), + new ExpectedNode(663806673, 51.765470, -0.229220), + new ExpectedNode(502550970, 51.765118, -0.233667), + new ExpectedNode(692887095, 51.766318, -0.229190), + new ExpectedNode(1685167376, 51.760411, -0.241161), + new ExpectedNode(175697821, 51.765000, -0.232204), + new ExpectedNode(677438877, 51.764126, -0.228303), + new ExpectedNode(175685111, 51.764882, -0.229966), + new ExpectedNode(647105131, 51.769022, -0.217223), + new ExpectedNode(240134267, 51.764217, -0.233120), + new ExpectedNode(691203111, 51.765755, -0.230230), + new ExpectedNode(1685167394, 51.761213, -0.240218), + new ExpectedNode(534873274, 51.763918, -0.236563), + new ExpectedNode(676945192, 51.765148, -0.230615), + new ExpectedNode(691203106, 51.764494, -0.233449), + new ExpectedNode(647105155, 51.769580, -0.232061), + new ExpectedNode(32950368, 51.769048, -0.232790), + new ExpectedNode(647105133, 51.769183, -0.216784), + new ExpectedNode(175683944, 51.763140, -0.232112), + new ExpectedNode(623540467, 51.765719, -0.225990), + new ExpectedNode(647225601, 51.762732, -0.231722), + new ExpectedNode(32953195, 51.761987, -0.231091), + new ExpectedNode(653970876, 51.763436, -0.229153), + new ExpectedNode(676945352, 51.765646, -0.228469), + new ExpectedNode(663806670, 51.765540, -0.228771), + new ExpectedNode(1709246676, 51.766438, -0.231121), + new ExpectedNode(647105047, 51.774057, -0.222895), + new ExpectedNode(175697862, 51.765004, -0.232747), + new ExpectedNode(647105145, 51.771007, -0.230355), + new ExpectedNode(647105167, 51.762860, -0.236278), + new ExpectedNode(1111758067, 51.771433, -0.216984), + new ExpectedNode(647105166, 51.767468, -0.234229), + new ExpectedNode(692887118, 51.766186, -0.228918), + new ExpectedNode(663806658, 51.765679, -0.228614), + new ExpectedNode(175685507, 51.765508, -0.229788), + new ExpectedNode(647224486, 51.766388, -0.228706), + new ExpectedNode(502552074, 51.766711, -0.229590), + new ExpectedNode(647105132, 51.768905, -0.216932), + new ExpectedNode(25365925, 51.766651, -0.233518), + new ExpectedNode(623540472, 51.765321, -0.225475), + new ExpectedNode(691202857, 51.766804, -0.231711), + new ExpectedNode(175686201, 51.765721, -0.228361), + new ExpectedNode(927070648, 51.763087, -0.232061), + new ExpectedNode(25365924, 51.767090, -0.233453), + new ExpectedNode(676945335, 51.765388, -0.228437), + new ExpectedNode(647105127, 51.769321, -0.219637), + new ExpectedNode(647105134, 51.769124, -0.216290), + new ExpectedNode(30983853, 51.764268, -0.233185), + new ExpectedNode(647105164, 51.767548, -0.233295), + new ExpectedNode(502552081, 51.766833, -0.233484), + new ExpectedNode(691202855, 51.766809, -0.231946), + new ExpectedNode(647057820, 51.765382, -0.226710), + new ExpectedNode(691202869, 51.767216, -0.231947), + new ExpectedNode(647105159, 51.768849, -0.232458), + new ExpectedNode(1739780291, 51.764890, -0.226086), + new ExpectedNode(676945267, 51.763905, -0.228040), + new ExpectedNode(663806664, 51.765444, -0.229274), + new ExpectedNode(647105143, 51.771399, -0.230034), + new ExpectedNode(691202858, 51.765928, -0.232698), + new ExpectedNode(1701110775, 51.766290, -0.228709), + new ExpectedNode(365548881, 51.763854, -0.232807), + new ExpectedNode(647224465, 51.765604, -0.226263), + new ExpectedNode(691202873, 51.766711, -0.232826), + new ExpectedNode(287659881, 51.766233, -0.228823), + new ExpectedNode(1685167328, 51.765389, -0.235803), + new ExpectedNode(1685167381, 51.762135, -0.238938), + new ExpectedNode(1685167371, 51.768683, -0.233758), + new ExpectedNode(1709246791, 51.765771, -0.229747), + new ExpectedNode(647105156, 51.769420, -0.232072), + new ExpectedNode(647105139, 51.773291, -0.221257), + new ExpectedNode(32953193, 51.763418, -0.232387), + new ExpectedNode(676945199, 51.765151, -0.230782), + new ExpectedNode(647105147, 51.770210, -0.231976), + new ExpectedNode(672628083, 51.764391, -0.225433), + new ExpectedNode(25365922, 51.768145, -0.233167), + new ExpectedNode(1709246741, 51.765960, -0.229886), + new ExpectedNode(647105153, 51.769673, -0.232265), + new ExpectedNode(30983851, 51.765372, -0.233546), + new ExpectedNode(691202863, 51.765224, -0.232225), + new ExpectedNode(691202838, 51.767798, -0.233387), + new ExpectedNode(175684459, 51.763370, -0.231564), + new ExpectedNode(1685167313, 51.762503, -0.238485), + new ExpectedNode(692945016, 51.765714, -0.230069), + new ExpectedNode(25365921, 51.768513, -0.232722), + new ExpectedNode(676945322, 51.765118, -0.229479), + new ExpectedNode(534873251, 51.763658, -0.236760), + new ExpectedNode(1685167341, 51.768171, -0.234063), + new ExpectedNode(691203110, 51.765769, -0.230874), + new ExpectedNode(676945292, 51.764506, -0.228754), + new ExpectedNode(1685167391, 51.761506, -0.239827), + new ExpectedNode(676945241, 51.763212, -0.229644), + new ExpectedNode(663806653, 51.765898, -0.228877), + new ExpectedNode(623624259, 51.764905, -0.234965), + new ExpectedNode(1685167373, 51.763777, -0.237235), + new ExpectedNode(676945320, 51.765375, -0.230143), + new ExpectedNode(240134268, 51.764403, -0.232382), + new ExpectedNode(676945316, 51.764949, -0.230532), + new ExpectedNode(623624154, 51.765244, -0.234365), + new ExpectedNode(647105142, 51.774147, -0.226321), + new ExpectedNode(1739780285, 51.764824, -0.226000), + new ExpectedNode(175697671, 51.765012, -0.233620), + new ExpectedNode(647224613, 51.764970, -0.229134), + new ExpectedNode(647105121, 51.769055, -0.221268), + new ExpectedNode(692887101, 51.766293, -0.228488), + new ExpectedNode(175683342, 51.763273, -0.229558), + new ExpectedNode(240134269, 51.765577, -0.230133), + new ExpectedNode(691203053, 51.766871, -0.230638), + new ExpectedNode(1697422651, 51.763725, -0.228467), + new ExpectedNode(534873285, 51.764110, -0.236786), + new ExpectedNode(647105148, 51.770131, -0.232104), + new ExpectedNode(647105165, 51.767482, -0.233317), + new ExpectedNode(534873185, 51.763403, -0.236752), + new ExpectedNode(175685104, 51.764391, -0.231506), + new ExpectedNode(647105163, 51.768079, -0.233048), + new ExpectedNode(651652536, 51.764591, -0.224432), + new ExpectedNode(647105115, 51.766990, -0.227373), + new ExpectedNode(677439944, 51.763332, -0.229790), + new ExpectedNode(647105162, 51.768232, -0.232866), + new ExpectedNode(676945319, 51.765218, -0.230449), + new ExpectedNode(1539682123, 51.769102, -0.232828), + new ExpectedNode(534873208, 51.763536, -0.236889), + new ExpectedNode(647105128, 51.769354, -0.219090), + new ExpectedNode(1739780280, 51.764758, -0.225914), + new ExpectedNode(175698323, 51.767216, -0.231110), + new ExpectedNode(676945189, 51.764650, -0.230926), + new ExpectedNode(1739780294, 51.764955, -0.224922), + new ExpectedNode(676945326, 51.765291, -0.229382), + new ExpectedNode(663806672, 51.765417, -0.229059), + new ExpectedNode(45169425, 51.769130, -0.233478), + new ExpectedNode(672663469, 51.765930, -0.229036), + new ExpectedNode(675146, 51.769270, -0.232860), + new ExpectedNode(691203054, 51.766658, -0.230273), + new ExpectedNode(1606957353, 51.760049, -0.241558), + new ExpectedNode(647105125, 51.769248, -0.220260), + new ExpectedNode(534874147, 51.765262, -0.235825), + new ExpectedNode(14713407, 51.765828, -0.227391), + new ExpectedNode(818056434, 51.766040, -0.233470), + new ExpectedNode(1111758069, 51.769198, -0.216444), + new ExpectedNode(175699187, 51.765663, -0.231004), + new ExpectedNode(175698155, 51.767389, -0.230809), + new ExpectedNode(691202861, 51.765516, -0.231002), + new ExpectedNode(651594517, 51.763745, -0.228419), + new ExpectedNode(691203051, 51.765901, -0.231217), + new ExpectedNode(647224485, 51.765127, -0.226399), + new ExpectedNode(1709246749, 51.765632, -0.230025), + new ExpectedNode(677440300, 51.762625, -0.231624), + new ExpectedNode(647105172, 51.764294, -0.233070), + new ExpectedNode(175686498, 51.765424, -0.228052), + new ExpectedNode(692944963, 51.764665, -0.233953), + new ExpectedNode(663806656, 51.765763, -0.228715), + new ExpectedNode(647105154, 51.769626, -0.232179), + new ExpectedNode(676945317, 51.765015, -0.230385), + new ExpectedNode(647105169, 51.763033, -0.235323), + new ExpectedNode(692945021, 51.766617, -0.229479), + new ExpectedNode(1709246789, 51.766231, -0.230173), + new ExpectedNode(175686499, 51.765976, -0.228635), + new ExpectedNode(691202866, 51.767110, -0.232955), + new ExpectedNode(1111758072, 51.769507, -0.216315), + new ExpectedNode(647105123, 51.769155, -0.220818), + new ExpectedNode(672663468, 51.765622, -0.228672), + new ExpectedNode(676945197, 51.765281, -0.230541), + new ExpectedNode(692945020, 51.766471, -0.229673), + new ExpectedNode(175697881, 51.764664, -0.232747), + new ExpectedNode(175685109, 51.764946, -0.230095), + new ExpectedNode(1685167304, 51.760787, -0.240738), + new ExpectedNode(692944951, 51.764943, -0.234254), + new ExpectedNode(692945019, 51.766225, -0.229673), + new ExpectedNode(676945334, 51.765467, -0.228255), + new ExpectedNode(175684463, 51.765445, -0.226790), + new ExpectedNode(692944957, 51.764651, -0.234168), + new ExpectedNode(647105144, 51.771332, -0.229905), + new ExpectedNode(691203055, 51.765928, -0.230187), + new ExpectedNode(676945331, 51.765589, -0.229749), + new ExpectedNode(672663474, 51.765638, -0.229315), + new ExpectedNode(647105146, 51.770283, -0.231836), + new ExpectedNode(534873171, 51.763005, -0.237147), + new ExpectedNode(647105157, 51.769307, -0.232308), + new ExpectedNode(676945327, 51.765347, -0.229744), + new ExpectedNode(675150, 51.766907, -0.229904), + new ExpectedNode(663806666, 51.765165, -0.228973), + new ExpectedNode(691202871, 51.766950, -0.232826), + new ExpectedNode(672663477, 51.765646, -0.228948), + new ExpectedNode(647105158, 51.769015, -0.232297), + new ExpectedNode(673784380, 51.762202, -0.231241), + new ExpectedNode(647105152, 51.769739, -0.232330), + new ExpectedNode(692945022, 51.766344, -0.228825), + new ExpectedNode(676945315, 51.764929, -0.230336), + new ExpectedNode(676945346, 51.765450, -0.228506), + new ExpectedNode(647105119, 51.768119, -0.223854), + new ExpectedNode(175698430, 51.766924, -0.231110), + new ExpectedNode(1685167387, 51.765901, -0.235408), + new ExpectedNode(175685910, 51.766003, -0.227820), + new ExpectedNode(820969139, 51.767836, -0.231358), + new ExpectedNode(647105102, 51.763883, -0.232727), + new ExpectedNode(675151, 51.766141, -0.228136), + new ExpectedNode(175698324, 51.766008, -0.231131), + new ExpectedNode(1685167282, 51.762958, -0.237989), + new ExpectedNode(502552090, 51.765557, -0.233577), + new ExpectedNode(623624155, 51.765449, -0.234590), + new ExpectedNode(267826070, 51.764017, -0.232970), + new ExpectedNode(25365930, 51.766791, -0.234972), + new ExpectedNode(676945195, 51.765156, -0.230570), + new ExpectedNode(1709246675, 51.766423, -0.230168), + new ExpectedNode(647105137, 51.774248, -0.218055), + new ExpectedNode(651652534, 51.764261, -0.225160), + new ExpectedNode(676945293, 51.764816, -0.229133), + new ExpectedNode(1692947499, 51.773860, -0.225851), + new ExpectedNode(623624257, 51.765396, -0.234075), + new ExpectedNode(175697824, 51.764998, -0.232032), + new ExpectedNode(672663478, 51.765575, -0.229104), + new ExpectedNode(1685167290, 51.763311, -0.237639), + new ExpectedNode(390911769, 51.766861, -0.229798), + new ExpectedNode(676945323, 51.765506, -0.229937), + new ExpectedNode(647105136, 51.773720, -0.217976), + new ExpectedNode(1539682039, 51.768036, -0.233265), + new ExpectedNode(691202860, 51.766247, -0.230595), + new ExpectedNode(1145410964, 51.769148, -0.232860), + new ExpectedNode(647105130, 51.769188, -0.217728), + new ExpectedNode(691203049, 51.766645, -0.234564), + new ExpectedNode(1539682089, 51.768368, -0.232938), + new ExpectedNode(175698550, 51.766911, -0.230809), + new ExpectedNode(623540479, 51.765560, -0.224961), + new ExpectedNode(677439941, 51.763240, -0.230472), + new ExpectedNode(25365927, 51.766333, -0.232681), + new ExpectedNode(647105135, 51.770431, -0.216476), + new ExpectedNode(30983852, 51.764773, -0.233577), + new ExpectedNode(647105150, 51.769938, -0.232265), + new ExpectedNode(623624261, 51.764407, -0.235985), + new ExpectedNode(647105149, 51.770024, -0.232212), + new ExpectedNode(677439946, 51.763219, -0.229690), + new ExpectedNode(691203109, 51.765671, -0.232912), + new ExpectedNode(647105171, 51.764248, -0.233242), + new ExpectedNode(1709246746, 51.766196, -0.230058), + new ExpectedNode(175685106, 51.764728, -0.230781), + new ExpectedNode(663806661, 51.765857, -0.228507), + new ExpectedNode(677439947, 51.764197, -0.228387), + new ExpectedNode(647105117, 51.767435, -0.226150), + new ExpectedNode(647105168, 51.762754, -0.235838), + new ExpectedNode(623624267, 51.764016, -0.233964), + new ExpectedNode(1709246737, 51.766423, -0.230671), + new ExpectedNode(175684462, 51.764271, -0.229245), + new ExpectedNode(175698551, 51.766539, -0.230166), + new ExpectedNode(675148, 51.768657, -0.232378), + new ExpectedNode(676945332, 51.764887, -0.229157), + new ExpectedNode(675149, 51.767913, -0.231459), + new ExpectedNode(692945018, 51.766178, -0.229758), + new ExpectedNode(623540483, 51.765155, -0.224456), + new ExpectedNode(676945350, 51.765533, -0.228712), + new ExpectedNode(175698975, 51.765729, -0.233577), + new ExpectedNode(175685102, 51.764176, -0.232069), + new ExpectedNode(676945347, 51.765417, -0.228572), + new ExpectedNode(534873262, 51.763775, -0.236889), + new ExpectedNode(25365931, 51.765437, -0.236066), + new ExpectedNode(672663470, 51.765668, -0.229614), + new ExpectedNode(647105138, 51.773941, -0.221418), + new ExpectedNode(647105151, 51.769819, -0.232340), + new ExpectedNode(32953194, 51.762232, -0.231263), + new ExpectedNode(1685167315, 51.766349, -0.235121), + new ExpectedNode(1111758071, 51.769058, -0.216775), + new ExpectedNode(691203098, 51.764324, -0.234279), + new ExpectedNode(175698553, 51.766253, -0.230172), + new ExpectedNode(1685167287, 51.767572, -0.234395), + new ExpectedNode(672663471, 51.765452, -0.229359), + new ExpectedNode(676945325, 51.765209, -0.229575), + new ExpectedNode(623624156, 51.765622, -0.234693), + new ExpectedNode(647105140, 51.773198, -0.222341), + new ExpectedNode(25365928, 51.766333, -0.232198), + new ExpectedNode(676945329, 51.765389, -0.229634), + new ExpectedNode(663806668, 51.765337, -0.228533), + new ExpectedNode(692944966, 51.764977, -0.233985), + new ExpectedNode(691203099, 51.764162, -0.234157), + new ExpectedNode(175685100, 51.764091, -0.232103), + new ExpectedNode(25365923, 51.767521, -0.233449), + new ExpectedNode(647105161, 51.767973, -0.232169), + new ExpectedNode(672663467, 51.765478, -0.228989), + new ExpectedNode(691202854, 51.766818, -0.232419) + }; + + URI testFileURL = NodeTest.class.getClassLoader().getResource("protobuf-test.osm.pbf").toURI(); + + Iterator<Entity> entities = new BlobSpliterator( + new RandomAccessFile(new File(testFileURL), "r"), + System.out::println) + .stream() + .flatMap(List::stream) + .filter(Node.class::isInstance) + .iterator(); + + int expectedIndex = 0; + while (entities.hasNext() && expectedIndex < expected.length) { + Node node = (Node) entities.next(); + ExpectedNode ex = expected[expectedIndex]; + + Assert.assertEquals(ex.id, node.getID()); + Assert.assertTrue("latitude", Math.abs(ex.latitude - node.getLatitude()) < 0.1); + Assert.assertTrue("longitude", + Math.abs(ex.longitude - node.getLongitude()) < 0.1); + + expectedIndex++; + } + } +} +\ No newline at end of file diff --git a/osm-protobuf/src/test/java/osm/protobuf/OrderTest.java b/osm-protobuf/src/test/java/osm/protobuf/OrderTest.java @@ -0,0 +1,38 @@ +package osm.protobuf; + +import java.io.File; +import java.io.RandomAccessFile; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.junit.Assert; +import org.junit.Test; + +import osm.message.Node; +import osm.message.Relation; +import osm.message.Way; +import osm.protobuf.common.CountingCollector; + +public class OrderTest { + @Test + public void test() throws Exception { + URI testFileURL = WayTest.class.getClassLoader().getResource("protobuf-test.osm.pbf").toURI(); + + List<Entry<Class<?>, Integer>> expectedCounts = List.of( + Map.entry(Node.class, 290), + Map.entry(Way.class, 44), + Map.entry(Relation.class, 5)); + + List<Entry<Class<?>, Integer>> counts = new BlobSpliterator( + new RandomAccessFile(new File(testFileURL), "r"), + System.out::println) + .stream() + .flatMap(List::stream) + .map(Object::getClass) + .collect(new CountingCollector()); + + Assert.assertEquals(expectedCounts, counts); + } +} diff --git a/osm-protobuf/src/test/java/osm/protobuf/Protobuf.java b/osm-protobuf/src/test/java/osm/protobuf/Protobuf.java @@ -1,475 +0,0 @@ -package osm.protobuf; - -import crosby.binary.Osmformat.*; -import crosby.binary.file.BlockInputStream; -import crosby.binary.file.BlockReaderAdapter; -import osm.common.TagMap; - -import org.junit.Assert; -import org.junit.Test; - -import java.io.InputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.List; - -/** - * Demonstrates how to read a file. Reads sample.pbf from the resources folder - * and prints details about it to the standard output. - * - * @author Michael Tandy - */ -public class Protobuf { - static record TestRow(Class<?> type, int id, TagMap tags, Point location, long[] children) { - } - - @Test - public void test() throws Exception { - String expected = ("" + - "Got header block.\n" + - "Dense node, ID 653970877 @ 51.763603,-0.228757\n" + - "Dense node, ID 647105170 @ 51.763591,-0.234465\n" + - "Dense node, ID 672663476 @ 51.765749,-0.229070\n" + - "Dense node, ID 241806356 @ 51.768945,-0.232662\n" + - "Dense node, ID 692945017 @ 51.766185,-0.230069\n" + - "Dense node, ID 1709246734 @ 51.766433,-0.230854\n" + - "Dense node, ID 175685506 @ 51.765169,-0.229374\n" + - "Dense node, ID 647105129 @ 51.769327,-0.218457\n" + - "Dense node, ID 647105160 @ 51.768192,-0.231686\n" + - "Dense node, ID 672663473 @ 51.765530,-0.229187\n" + - "Dense node, ID 647105141 @ 51.773204,-0.222598\n" + - "Dense node, ID 25365926 @ 51.766340,-0.233556\n" + - "Dense node, ID 1685167296 @ 51.766924,-0.234783\n" + - "Dense node, ID 677439943 @ 51.763178,-0.230230\n" + - "Dense node, ID 1701110757 @ 51.766400,-0.228489\n" + - "Dense node, ID 663806673 @ 51.765470,-0.229220\n" + - "Dense node, ID 502550970 @ 51.765118,-0.233667\n" + - "Dense node, ID 692887095 @ 51.766318,-0.229190\n" + - "Dense node, ID 1685167376 @ 51.760411,-0.241161\n" + - "Dense node, ID 175697821 @ 51.765000,-0.232204\n" + - "Dense node, ID 677438877 @ 51.764126,-0.228303\n" + - "Dense node, ID 175685111 @ 51.764882,-0.229966\n" + - "Dense node, ID 647105131 @ 51.769022,-0.217223\n" + - "Dense node, ID 240134267 @ 51.764217,-0.233120\n" + - "Dense node, ID 691203111 @ 51.765755,-0.230230\n" + - "Dense node, ID 1685167394 @ 51.761213,-0.240218\n" + - "Dense node, ID 534873274 @ 51.763918,-0.236563\n" + - "Dense node, ID 676945192 @ 51.765148,-0.230615\n" + - "Dense node, ID 691203106 @ 51.764494,-0.233449\n" + - "Dense node, ID 647105155 @ 51.769580,-0.232061\n" + - "Dense node, ID 32950368 @ 51.769048,-0.232790\n" + - "Dense node, ID 647105133 @ 51.769183,-0.216784\n" + - "Dense node, ID 175683944 @ 51.763140,-0.232112\n" + - "Dense node, ID 623540467 @ 51.765719,-0.225990\n" + - "Dense node, ID 647225601 @ 51.762732,-0.231722\n" + - "Dense node, ID 32953195 @ 51.761987,-0.231091\n" + - "Dense node, ID 653970876 @ 51.763436,-0.229153\n" + - "Dense node, ID 676945352 @ 51.765646,-0.228469\n" + - "Dense node, ID 663806670 @ 51.765540,-0.228771\n" + - "Dense node, ID 1709246676 @ 51.766438,-0.231121\n" + - "Dense node, ID 647105047 @ 51.774057,-0.222895\n" + - "Dense node, ID 175697862 @ 51.765004,-0.232747\n" + - "Dense node, ID 647105145 @ 51.771007,-0.230355\n" + - "Dense node, ID 647105167 @ 51.762860,-0.236278\n" + - "Dense node, ID 1111758067 @ 51.771433,-0.216984\n" + - "Dense node, ID 647105166 @ 51.767468,-0.234229\n" + - "Dense node, ID 692887118 @ 51.766186,-0.228918\n" + - "Dense node, ID 663806658 @ 51.765679,-0.228614\n" + - "Dense node, ID 175685507 @ 51.765508,-0.229788\n" + - "Dense node, ID 647224486 @ 51.766388,-0.228706\n" + - "Dense node, ID 502552074 @ 51.766711,-0.229590\n" + - "Dense node, ID 647105132 @ 51.768905,-0.216932\n" + - "Dense node, ID 25365925 @ 51.766651,-0.233518\n" + - "Dense node, ID 623540472 @ 51.765321,-0.225475\n" + - "Dense node, ID 691202857 @ 51.766804,-0.231711\n" + - "Dense node, ID 175686201 @ 51.765721,-0.228361\n" + - "Dense node, ID 927070648 @ 51.763087,-0.232061\n" + - "Dense node, ID 25365924 @ 51.767090,-0.233453\n" + - "Dense node, ID 676945335 @ 51.765388,-0.228437\n" + - "Dense node, ID 647105127 @ 51.769321,-0.219637\n" + - "Dense node, ID 647105134 @ 51.769124,-0.216290\n" + - "Dense node, ID 30983853 @ 51.764268,-0.233185\n" + - "Dense node, ID 647105164 @ 51.767548,-0.233295\n" + - "Dense node, ID 502552081 @ 51.766833,-0.233484\n" + - "Dense node, ID 691202855 @ 51.766809,-0.231946\n" + - "Dense node, ID 647057820 @ 51.765382,-0.226710\n" + - "Dense node, ID 691202869 @ 51.767216,-0.231947\n" + - "Dense node, ID 647105159 @ 51.768849,-0.232458\n" + - "Dense node, ID 1739780291 @ 51.764890,-0.226086\n" + - "Dense node, ID 676945267 @ 51.763905,-0.228040\n" + - "Dense node, ID 663806664 @ 51.765444,-0.229274\n" + - "Dense node, ID 647105143 @ 51.771399,-0.230034\n" + - "Dense node, ID 691202858 @ 51.765928,-0.232698\n" + - "Dense node, ID 1701110775 @ 51.766290,-0.228709\n" + - "Dense node, ID 365548881 @ 51.763854,-0.232807\n" + - "Dense node, ID 647224465 @ 51.765604,-0.226263\n" + - "Dense node, ID 691202873 @ 51.766711,-0.232826\n" + - "Dense node, ID 287659881 @ 51.766233,-0.228823\n" + - "Dense node, ID 1685167328 @ 51.765389,-0.235803\n" + - "Dense node, ID 1685167381 @ 51.762135,-0.238938\n" + - "Dense node, ID 1685167371 @ 51.768683,-0.233758\n" + - "Dense node, ID 1709246791 @ 51.765771,-0.229747\n" + - "Dense node, ID 647105156 @ 51.769420,-0.232072\n" + - "Dense node, ID 647105139 @ 51.773291,-0.221257\n" + - "Dense node, ID 32953193 @ 51.763418,-0.232387\n" + - "Dense node, ID 676945199 @ 51.765151,-0.230782\n" + - "Dense node, ID 647105147 @ 51.770210,-0.231976\n" + - "Dense node, ID 672628083 @ 51.764391,-0.225433\n" + - "Dense node, ID 25365922 @ 51.768145,-0.233167\n" + - "Dense node, ID 1709246741 @ 51.765960,-0.229886\n" + - "Dense node, ID 647105153 @ 51.769673,-0.232265\n" + - "Dense node, ID 30983851 @ 51.765372,-0.233546\n" + - "Dense node, ID 691202863 @ 51.765224,-0.232225\n" + - "Dense node, ID 691202838 @ 51.767798,-0.233387\n" + - "Dense node, ID 175684459 @ 51.763370,-0.231564\n" + - "Dense node, ID 1685167313 @ 51.762503,-0.238485\n" + - "Dense node, ID 692945016 @ 51.765714,-0.230069\n" + - "Dense node, ID 25365921 @ 51.768513,-0.232722\n" + - "Dense node, ID 676945322 @ 51.765118,-0.229479\n" + - "Dense node, ID 534873251 @ 51.763658,-0.236760\n" + - "Dense node, ID 1685167341 @ 51.768171,-0.234063\n" + - "Dense node, ID 691203110 @ 51.765769,-0.230874\n" + - "Dense node, ID 676945292 @ 51.764506,-0.228754\n" + - "Dense node, ID 1685167391 @ 51.761506,-0.239827\n" + - "Dense node, ID 676945241 @ 51.763212,-0.229644\n" + - "Dense node, ID 663806653 @ 51.765898,-0.228877\n" + - "Dense node, ID 623624259 @ 51.764905,-0.234965\n" + - "Dense node, ID 1685167373 @ 51.763777,-0.237235\n" + - "Dense node, ID 676945320 @ 51.765375,-0.230143\n" + - "Dense node, ID 240134268 @ 51.764403,-0.232382\n" + - "Dense node, ID 676945316 @ 51.764949,-0.230532\n" + - "Dense node, ID 623624154 @ 51.765244,-0.234365\n" + - "Dense node, ID 647105142 @ 51.774147,-0.226321\n" + - "Dense node, ID 1739780285 @ 51.764824,-0.226000\n" + - "Dense node, ID 175697671 @ 51.765012,-0.233620\n" + - "Dense node, ID 647224613 @ 51.764970,-0.229134\n" + - "Dense node, ID 647105121 @ 51.769055,-0.221268\n" + - "Dense node, ID 692887101 @ 51.766293,-0.228488\n" + - "Dense node, ID 175683342 @ 51.763273,-0.229558\n" + - "Dense node, ID 240134269 @ 51.765577,-0.230133\n" + - "Dense node, ID 691203053 @ 51.766871,-0.230638\n" + - "Dense node, ID 1697422651 @ 51.763725,-0.228467\n" + - "Dense node, ID 534873285 @ 51.764110,-0.236786\n" + - "Dense node, ID 647105148 @ 51.770131,-0.232104\n" + - "Dense node, ID 647105165 @ 51.767482,-0.233317\n" + - "Dense node, ID 534873185 @ 51.763403,-0.236752\n" + - "Dense node, ID 175685104 @ 51.764391,-0.231506\n" + - "Dense node, ID 647105163 @ 51.768079,-0.233048\n" + - "Dense node, ID 651652536 @ 51.764591,-0.224432\n" + - "Dense node, ID 647105115 @ 51.766990,-0.227373\n" + - "Dense node, ID 677439944 @ 51.763332,-0.229790\n" + - "Dense node, ID 647105162 @ 51.768232,-0.232866\n" + - "Dense node, ID 676945319 @ 51.765218,-0.230449\n" + - "Dense node, ID 1539682123 @ 51.769102,-0.232828\n" + - "Dense node, ID 534873208 @ 51.763536,-0.236889\n" + - "Dense node, ID 647105128 @ 51.769354,-0.219090\n" + - "Dense node, ID 1739780280 @ 51.764758,-0.225914\n" + - "Dense node, ID 175698323 @ 51.767216,-0.231110\n" + - "Dense node, ID 676945189 @ 51.764650,-0.230926\n" + - "Dense node, ID 1739780294 @ 51.764955,-0.224922\n" + - "Dense node, ID 676945326 @ 51.765291,-0.229382\n" + - "Dense node, ID 663806672 @ 51.765417,-0.229059\n" + - "Dense node, ID 45169425 @ 51.769130,-0.233478\n" + - "Dense node, ID 672663469 @ 51.765930,-0.229036\n" + - "Dense node, ID 675146 @ 51.769270,-0.232860\n" + - "Dense node, ID 691203054 @ 51.766658,-0.230273\n" + - "Dense node, ID 1606957353 @ 51.760049,-0.241558\n" + - "Dense node, ID 647105125 @ 51.769248,-0.220260\n" + - "Dense node, ID 534874147 @ 51.765262,-0.235825\n" + - "Dense node, ID 14713407 @ 51.765828,-0.227391\n" + - "Dense node, ID 818056434 @ 51.766040,-0.233470\n" + - "Dense node, ID 1111758069 @ 51.769198,-0.216444\n" + - "Dense node, ID 175699187 @ 51.765663,-0.231004\n" + - "Dense node, ID 175698155 @ 51.767389,-0.230809\n" + - "Dense node, ID 691202861 @ 51.765516,-0.231002\n" + - "Dense node, ID 651594517 @ 51.763745,-0.228419\n" + - "Dense node, ID 691203051 @ 51.765901,-0.231217\n" + - "Dense node, ID 647224485 @ 51.765127,-0.226399\n" + - "Dense node, ID 1709246749 @ 51.765632,-0.230025\n" + - "Dense node, ID 677440300 @ 51.762625,-0.231624\n" + - "Dense node, ID 647105172 @ 51.764294,-0.233070\n" + - "Dense node, ID 175686498 @ 51.765424,-0.228052\n" + - "Dense node, ID 692944963 @ 51.764665,-0.233953\n" + - "Dense node, ID 663806656 @ 51.765763,-0.228715\n" + - "Dense node, ID 647105154 @ 51.769626,-0.232179\n" + - "Dense node, ID 676945317 @ 51.765015,-0.230385\n" + - "Dense node, ID 647105169 @ 51.763033,-0.235323\n" + - "Dense node, ID 692945021 @ 51.766617,-0.229479\n" + - "Dense node, ID 1709246789 @ 51.766231,-0.230173\n" + - "Dense node, ID 175686499 @ 51.765976,-0.228635\n" + - "Dense node, ID 691202866 @ 51.767110,-0.232955\n" + - "Dense node, ID 1111758072 @ 51.769507,-0.216315\n" + - "Dense node, ID 647105123 @ 51.769155,-0.220818\n" + - "Dense node, ID 672663468 @ 51.765622,-0.228672\n" + - "Dense node, ID 676945197 @ 51.765281,-0.230541\n" + - "Dense node, ID 692945020 @ 51.766471,-0.229673\n" + - "Dense node, ID 175697881 @ 51.764664,-0.232747\n" + - "Dense node, ID 175685109 @ 51.764946,-0.230095\n" + - "Dense node, ID 1685167304 @ 51.760787,-0.240738\n" + - "Dense node, ID 692944951 @ 51.764943,-0.234254\n" + - "Dense node, ID 692945019 @ 51.766225,-0.229673\n" + - "Dense node, ID 676945334 @ 51.765467,-0.228255\n" + - "Dense node, ID 175684463 @ 51.765445,-0.226790\n" + - "Dense node, ID 692944957 @ 51.764651,-0.234168\n" + - "Dense node, ID 647105144 @ 51.771332,-0.229905\n" + - "Dense node, ID 691203055 @ 51.765928,-0.230187\n" + - "Dense node, ID 676945331 @ 51.765589,-0.229749\n" + - "Dense node, ID 672663474 @ 51.765638,-0.229315\n" + - "Dense node, ID 647105146 @ 51.770283,-0.231836\n" + - "Dense node, ID 534873171 @ 51.763005,-0.237147\n" + - "Dense node, ID 647105157 @ 51.769307,-0.232308\n" + - "Dense node, ID 676945327 @ 51.765347,-0.229744\n" + - "Dense node, ID 675150 @ 51.766907,-0.229904\n" + - "Dense node, ID 663806666 @ 51.765165,-0.228973\n" + - "Dense node, ID 691202871 @ 51.766950,-0.232826\n" + - "Dense node, ID 672663477 @ 51.765646,-0.228948\n" + - "Dense node, ID 647105158 @ 51.769015,-0.232297\n" + - "Dense node, ID 673784380 @ 51.762202,-0.231241\n" + - "Dense node, ID 647105152 @ 51.769739,-0.232330\n" + - "Dense node, ID 692945022 @ 51.766344,-0.228825\n" + - "Dense node, ID 676945315 @ 51.764929,-0.230336\n" + - "Dense node, ID 676945346 @ 51.765450,-0.228506\n" + - "Dense node, ID 647105119 @ 51.768119,-0.223854\n" + - "Dense node, ID 175698430 @ 51.766924,-0.231110\n" + - "Dense node, ID 1685167387 @ 51.765901,-0.235408\n" + - "Dense node, ID 175685910 @ 51.766003,-0.227820\n" + - "Dense node, ID 820969139 @ 51.767836,-0.231358\n" + - "Dense node, ID 647105102 @ 51.763883,-0.232727\n" + - "Dense node, ID 675151 @ 51.766141,-0.228136\n" + - "Dense node, ID 175698324 @ 51.766008,-0.231131\n" + - "Dense node, ID 1685167282 @ 51.762958,-0.237989\n" + - "Dense node, ID 502552090 @ 51.765557,-0.233577\n" + - "Dense node, ID 623624155 @ 51.765449,-0.234590\n" + - "Dense node, ID 267826070 @ 51.764017,-0.232970\n" + - "Dense node, ID 25365930 @ 51.766791,-0.234972\n" + - "Dense node, ID 676945195 @ 51.765156,-0.230570\n" + - "Dense node, ID 1709246675 @ 51.766423,-0.230168\n" + - "Dense node, ID 647105137 @ 51.774248,-0.218055\n" + - "Dense node, ID 651652534 @ 51.764261,-0.225160\n" + - "Dense node, ID 676945293 @ 51.764816,-0.229133\n" + - "Dense node, ID 1692947499 @ 51.773860,-0.225851\n" + - "Dense node, ID 623624257 @ 51.765396,-0.234075\n" + - "Dense node, ID 175697824 @ 51.764998,-0.232032\n" + - "Dense node, ID 672663478 @ 51.765575,-0.229104\n" + - "Dense node, ID 1685167290 @ 51.763311,-0.237639\n" + - "Dense node, ID 390911769 @ 51.766861,-0.229798\n" + - "Dense node, ID 676945323 @ 51.765506,-0.229937\n" + - "Dense node, ID 647105136 @ 51.773720,-0.217976\n" + - "Dense node, ID 1539682039 @ 51.768036,-0.233265\n" + - "Dense node, ID 691202860 @ 51.766247,-0.230595\n" + - "Dense node, ID 1145410964 @ 51.769148,-0.232860\n" + - "Dense node, ID 647105130 @ 51.769188,-0.217728\n" + - "Dense node, ID 691203049 @ 51.766645,-0.234564\n" + - "Dense node, ID 1539682089 @ 51.768368,-0.232938\n" + - "Dense node, ID 175698550 @ 51.766911,-0.230809\n" + - "Dense node, ID 623540479 @ 51.765560,-0.224961\n" + - "Dense node, ID 677439941 @ 51.763240,-0.230472\n" + - "Dense node, ID 25365927 @ 51.766333,-0.232681\n" + - "Dense node, ID 647105135 @ 51.770431,-0.216476\n" + - "Dense node, ID 30983852 @ 51.764773,-0.233577\n" + - "Dense node, ID 647105150 @ 51.769938,-0.232265\n" + - "Dense node, ID 623624261 @ 51.764407,-0.235985\n" + - "Dense node, ID 647105149 @ 51.770024,-0.232212\n" + - "Dense node, ID 677439946 @ 51.763219,-0.229690\n" + - "Dense node, ID 691203109 @ 51.765671,-0.232912\n" + - "Dense node, ID 647105171 @ 51.764248,-0.233242\n" + - "Dense node, ID 1709246746 @ 51.766196,-0.230058\n" + - "Dense node, ID 175685106 @ 51.764728,-0.230781\n" + - "Dense node, ID 663806661 @ 51.765857,-0.228507\n" + - "Dense node, ID 677439947 @ 51.764197,-0.228387\n" + - "Dense node, ID 647105117 @ 51.767435,-0.226150\n" + - "Dense node, ID 647105168 @ 51.762754,-0.235838\n" + - "Dense node, ID 623624267 @ 51.764016,-0.233964\n" + - "Dense node, ID 1709246737 @ 51.766423,-0.230671\n" + - "Dense node, ID 175684462 @ 51.764271,-0.229245\n" + - "Dense node, ID 175698551 @ 51.766539,-0.230166\n" + - "Dense node, ID 675148 @ 51.768657,-0.232378\n" + - "Dense node, ID 676945332 @ 51.764887,-0.229157\n" + - "Dense node, ID 675149 @ 51.767913,-0.231459\n" + - "Dense node, ID 692945018 @ 51.766178,-0.229758\n" + - "Dense node, ID 623540483 @ 51.765155,-0.224456\n" + - "Dense node, ID 676945350 @ 51.765533,-0.228712\n" + - "Dense node, ID 175698975 @ 51.765729,-0.233577\n" + - "Dense node, ID 175685102 @ 51.764176,-0.232069\n" + - "Dense node, ID 676945347 @ 51.765417,-0.228572\n" + - "Dense node, ID 534873262 @ 51.763775,-0.236889\n" + - "Dense node, ID 25365931 @ 51.765437,-0.236066\n" + - "Dense node, ID 672663470 @ 51.765668,-0.229614\n" + - "Dense node, ID 647105138 @ 51.773941,-0.221418\n" + - "Dense node, ID 647105151 @ 51.769819,-0.232340\n" + - "Dense node, ID 32953194 @ 51.762232,-0.231263\n" + - "Dense node, ID 1685167315 @ 51.766349,-0.235121\n" + - "Dense node, ID 1111758071 @ 51.769058,-0.216775\n" + - "Dense node, ID 691203098 @ 51.764324,-0.234279\n" + - "Dense node, ID 175698553 @ 51.766253,-0.230172\n" + - "Dense node, ID 1685167287 @ 51.767572,-0.234395\n" + - "Dense node, ID 672663471 @ 51.765452,-0.229359\n" + - "Dense node, ID 676945325 @ 51.765209,-0.229575\n" + - "Dense node, ID 623624156 @ 51.765622,-0.234693\n" + - "Dense node, ID 647105140 @ 51.773198,-0.222341\n" + - "Dense node, ID 25365928 @ 51.766333,-0.232198\n" + - "Dense node, ID 676945329 @ 51.765389,-0.229634\n" + - "Dense node, ID 663806668 @ 51.765337,-0.228533\n" + - "Dense node, ID 692944966 @ 51.764977,-0.233985\n" + - "Dense node, ID 691203099 @ 51.764162,-0.234157\n" + - "Dense node, ID 175685100 @ 51.764091,-0.232103\n" + - "Dense node, ID 25365923 @ 51.767521,-0.233449\n" + - "Dense node, ID 647105161 @ 51.767973,-0.232169\n" + - "Dense node, ID 672663467 @ 51.765478,-0.228989\n" + - "Dense node, ID 691202854 @ 51.766818,-0.232419\n" + - "Way ID 158788812\n" + - " Nodes: 1709246789 1709246746 1709246741 1709246791 \n" + - " Key=value pairs: highway=footway \n" + - "Way ID 53588781\n" + - " Nodes: 676945323 676945327 676945325 676945326 676945331 676945323 \n" + - " Key=value pairs: landuse=garages source=survey \n" + - "Way ID 158788810\n" + - " Nodes: 1709246675 1709246737 1709246734 1709246676 \n" + - " Key=value pairs: highway=footway \n" + - "Way ID 156255508\n" + - " Nodes: 45169425 1685167371 1685167341 1685167287 1685167296 1685167315 1685167387 1685167328 1685167373 1685167290 1685167282 1685167313 1685167381 1685167391 1685167394 1685167304 1685167376 1606957353 \n" - + - " Key=value pairs: carriageway_ref=A highway=motorway lanes=3 layer=-1 lit=yes maxspeed=national name=Hatfield Tunnel oneway=yes ref=A1(M) source:maxspeed=local_knowledge tunnel=yes \n" - + - "Way ID 54932035\n" + - " Nodes: 691202854 691202855 691202857 \n" + - " Key=value pairs: highway=residential name=Jasmine Gardens source=OS_OpenData_StreetView \n" + - "Way ID 16946553\n" + - " Nodes: 175697671 175697862 175697821 175697824 \n" + - " Key=value pairs: highway=residential name=Oak Tree Close \n" + - "Way ID 52083876\n" + - " Nodes: 663806653 663806656 663806658 \n" + - " Key=value pairs: highway=service \n" + - "Way ID 55081202\n" + - " Nodes: 692944951 692944957 692944963 692944966 692944951 \n" + - " Key=value pairs: leisure=common source=yahoo \n" + - "Way ID 16946600\n" + - " Nodes: 175698430 175698550 691203053 691203054 175698551 1709246675 175698553 1709246789 691203055 \n" - + - " Key=value pairs: highway=residential name=Harmony Close source=OS_OpenData_StreetView \n" + - "Way ID 16946559\n" + - " Nodes: 175697862 175697881 \n" + - " Key=value pairs: created_by=Potlatch 0.5d highway=residential name=Oak Tree Close \n" + - "Way ID 16945846\n" + - " Nodes: 175684459 175685100 175685102 175685104 676945189 175685106 676945315 175685109 175685111 175684462 \n" - + - " Key=value pairs: highway=residential name=Stockbreach Close \n" + - "Way ID 54932037\n" + - " Nodes: 175698553 691202860 \n" + - " Key=value pairs: highway=residential name=Harmony Close source=OS_OpenData_StreetView \n" + - "Way ID 16946584\n" + - " Nodes: 175698155 175698323 175698430 1709246676 175698324 691203051 \n" + - " Key=value pairs: highway=residential name=The Minims source=OS_OpenData_StreetView \n" + - "Way ID 8361329\n" + - " Nodes: 25365926 25365927 25365928 \n" + - " Key=value pairs: highway=residential name=The Paddock source=OS_OpenData_StreetView \n" + - "Way ID 16945926\n" + - " Nodes: 175686498 175686201 663806661 175686499 \n" + - " Key=value pairs: highway=residential name=Wellfield Close \n" + - "Way ID 52083878\n" + - " Nodes: 663806664 663806666 663806668 663806670 663806672 663806673 663806664 \n" + - " Key=value pairs: leisure=common source=yahoo \n" + - "Way ID 3084923\n" + - " Nodes: 675146 1145410964 1539682123 32950368 241806356 675148 675149 820969139 175698155 675150 390911769 647224486 675151 175685910 14713407 175684463 647057820 647224485 1739780291 \n" - + - " Key=value pairs: abutters=residential highway=secondary name=Wellfield Road ref=B197 \n" + - "Way ID 8361331\n" + - " Nodes: 25365925 691203049 25365930 25365931 \n" + - " Key=value pairs: highway=residential name=Walsingham Close source=OS_OpenData_StreetView \n" + - "Way ID 54932038\n" + - " Nodes: 175699187 691202861 \n" + - " Key=value pairs: highway=residential name=Middlefield source=OS_OpenData_StreetView \n" + - "Way ID 16946620\n" + - " Nodes: 175698975 691203109 175699187 691203110 691203111 \n" + - " Key=value pairs: highway=residential name=Middlefield source=OS_OpenData_StreetView \n" + - "Way ID 157868709\n" + - " Nodes: 1701110757 1701110775 \n" + - " Key=value pairs: bridge=yes cycleway=track highway=cycleway layer=1 name=Alban Way ncn_ref=61 railway=abandoned \n" - + - "Way ID 52083877\n" + - " Nodes: 663806656 663806661 \n" + - " Key=value pairs: highway=service \n" + - "Way ID 53588764\n" + - " Nodes: 676945267 677438877 677439947 676945292 676945293 676945332 647224613 175686498 \n" + - " Key=value pairs: highway=footway \n" + - "Way ID 49161822\n" + - " Nodes: 30983851 623624257 623624154 623624259 623624261 \n" + - " Key=value pairs: highway=residential name=Worcester Road \n" + - "Way ID 49161823\n" + - " Nodes: 623624259 691203098 691203099 623624267 \n" + - " Key=value pairs: highway=residential name=Ely Close \n" + - "Way ID 53588782\n" + - " Nodes: 676945327 676945329 \n" + - " Key=value pairs: bicycle=no highway=footway source=survey \n" + - "Way ID 54932044\n" + - " Nodes: 691202869 691202855 \n" + - " Key=value pairs: highway=residential name=Jasmine Gardens source=OS_OpenData_StreetView \n" + - "Way ID 49161817\n" + - " Nodes: 623624154 623624155 623624156 \n" + - " Key=value pairs: highway=residential name=Malvern Close \n" + - "Way ID 53588780\n" + - " Nodes: 676945350 676945352 676945334 676945335 676945346 676945347 676945350 \n" + - " Key=value pairs: building=yes name=Friendship House source=survey \n" + - "Way ID 53588749\n" + - " Nodes: 676945199 676945316 676945317 676945195 676945319 676945197 676945199 \n" + - " Key=value pairs: landuse=garages source=survey \n" + - "Way ID 4673618\n" + - " Nodes: 675148 25365921 1539682089 25365922 1539682039 691202838 25365923 25365924 25365925 25365926 175698975 30983851 175697671 30983852 691203106 30983853 240134267 267826070 365548881 32953193 175683944 927070648 647225601 677440300 32953194 673784380 32953195 \n" - + - " Key=value pairs: highway=tertiary name=Lemsford Road \n" + - "Way ID 53638158\n" + - " Nodes: 677439941 677439943 677439944 677439946 676945241 175683342 653970876 653970877 1697422651 651594517 676945267 677438877 677439947 647224485 1739780291 1739780285 1739780280 672628083 651652534 651652536 1739780294 623540483 623540479 623540472 623540467 647224465 647057820 175684463 14713407 175685910 675151 692887101 647224486 647105115 647105117 647105119 647105121 647105123 647105125 647105127 647105128 647105129 647105130 647105131 647105132 1111758071 647105133 1111758069 647105134 1111758072 647105135 1111758067 647105136 647105137 647105138 647105139 647105140 647105141 647105047 1692947499 647105142 647105143 647105144 647105145 647105146 647105147 647105148 647105149 647105150 647105151 647105152 647105153 647105154 647105155 647105156 647105157 647105158 647105159 647105160 647105161 647105162 647105163 647105164 647105165 647105166 534874147 534873285 534873274 534873262 534873251 534873208 534873185 534873171 647105167 647105168 647105169 647105170 647105171 647105172 647105102 647225601 677439941 \n" - + - " Key=value pairs: landuse=residential \n" + - "Way ID 53588748\n" + - " Nodes: 676945322 676945325 676945327 676945320 676945192 676945315 \n" + - " Key=value pairs: bicycle=no highway=footway source=survey \n" + - "Way ID 50772651\n" + - " Nodes: 175685111 676945322 175685506 647224613 \n" + - " Key=value pairs: highway=residential name=Town Fields \n" + - "Way ID 158788824\n" + - " Nodes: 175685507 1709246749 \n" + - " Key=value pairs: highway=footway \n" + - "Way ID 54932042\n" + - " Nodes: 691202866 691202871 691202873 \n" + - " Key=value pairs: highway=residential name=Jasmine Gardens source=OS_OpenData_StreetView \n" + - "Way ID 53152061\n" + - " Nodes: 672663467 672663468 672663469 672663470 672663471 672663473 672663474 672663476 672663477 672663478 672663467 \n" - + - " Key=value pairs: amenity=retirement _home building=yes name=Greenacres source:area=yahoo source:name=survey \n" - + - "Way ID 53638215\n" + - " Nodes: 175685506 676945329 175685507 \n" + - " Key=value pairs: highway=service source=survey \n" + - "Way ID 157868710\n" + - " Nodes: 1701110775 287659881 692887118 1709246791 1709246749 240134269 240134268 \n" + - " Key=value pairs: cycleway=track highway=cycleway name=Alban Way ncn_ref=61 railway=abandoned \n" + - "Way ID 54932036\n" + - " Nodes: 25365927 691202858 \n" + - " Key=value pairs: highway=residential name=The Paddock source=OS_OpenData_StreetView \n" + - "Way ID 54932039\n" + - " Nodes: 175697821 691202863 \n" + - " Key=value pairs: highway=residential name=Oak Tree Close source=OS_OpenData_StreetView \n" + - "Way ID 55081204\n" + - " Nodes: 692945016 692945017 692945018 692945019 692945020 692945021 692945022 692945016 \n" + - " Key=value pairs: leisure=common \n" + - "Way ID 55071941\n" + - " Nodes: 692887118 692887101 \n" + - " Key=value pairs: foot=yes highway=footway \n" + - "Way ID 157868707\n" + - " Nodes: 240134268 240134269 1709246749 1709246791 692887118 287659881 \n" + - " Key=value pairs: cycleway=track highway=cycleway name=Alban Way ncn_ref=61 \n" + - "Got some relations to parse.\n" + - "Complete!\n").replace("\n", System.lineSeparator()); - - new BlobSpliterator(); - try (InputStream input = Protobuf.class.getResource("/sample.pbf"); - StringWriter stringWriter = new StringWriter(); - PrintWriter printWriter = new PrintWriter(stringWriter)) { - BlockReaderAdapter brad = new TestBinaryParser(printWriter); - new BlockInputStream(input, brad).process(); - - Assert.assertEquals(expected, stringWriter.toString()); - } - } -} -\ No newline at end of file diff --git a/osm-protobuf/src/test/java/osm/protobuf/RelationTest.java b/osm-protobuf/src/test/java/osm/protobuf/RelationTest.java @@ -0,0 +1,123 @@ +package osm.protobuf; + +import org.junit.Test; + +import java.io.File; +import java.io.RandomAccessFile; +import java.net.URI; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.junit.Assert; + +import osm.message.Entity; +import osm.message.Relation; +import osm.message.Relation.RelationMemberType; + +/** + * Demonstrates how to read a file. Reads sample.pbf from the resources folder + * and prints details about it to the standard output. + * + * @author Michael Tandy + */ +public class RelationTest { + public static record ExpectedRelation(long id, List<RelationMemberType> memberTypes, List<Long> memberIDs, + List<String> tags) { + public Map<String, String> tagMap() { + Map<String, String> map = new HashMap<>(); + for (int i = 0; i < tags.size(); i += 2) { + map.put(tags.get(i), tags.get(i + 1)); + } + return map; + } + } + + @Test + public void test() throws Exception { + ExpectedRelation[] expected = new ExpectedRelation[] { + new ExpectedRelation(21855, List.of(RelationMemberType.WAY, RelationMemberType.WAY), + List.of(156255508l, 156255507l), + List.of("name", "Hatfield Tunnel", "type", "tunnel")), + new ExpectedRelation(31640, + Stream.generate(() -> RelationMemberType.WAY).limit(234).toList(), + List.of(24541150l, 25896432l, 25896435l, 136990875l, 25896366l, 25896438l, 136990877l, + 22329168l, 136990870l, 136990880l, 136990882l, 3220127l, 136990873l, 136990872l, + 121267851l, 121267847l, 4515378l, 8145197l, 19745207l, 19745206l, 4518667l, 113946003l, + 136990884l, 8126872l, 1019866l, 1935841l, 1935842l, 2837807l, 3617718l, 3617750l, + 157868716l, 157868712l, 4232426l, 4232450l, 4232523l, 4234302l, 4234304l, 140969196l, + 4274177l, 4288798l, 4288804l, 4288817l, 115708422l, 4290564l, 4290566l, 146405866l, + 4290567l, 4290569l, 104494931l, 37726893l, 4300013l, 4327072l, 4385337l, 4515181l, + 4719529l, 4719530l, 5924137l, 136990881l, 135775543l, 135775531l, 6006524l, 9227120l, + 9228211l, 9228215l, 9228221l, 9228224l, 9265930l, 9362548l, 10943268l, 10943269l, + 10944242l, 10989591l, 10991833l, 10996488l, 11070521l, 11071125l, 71403686l, 11071127l, + 11071128l, 11071130l, 11071828l, 11071937l, 11071938l, 11071941l, 11072077l, 11072080l, + 15091285l, 15091286l, 15091321l, 15091324l, 15091328l, 15091364l, 15091378l, 73271466l, + 73271488l, 58718382l, 15091546l, 15091557l, 15091559l, 15091596l, 15091603l, 15091606l, + 48827325l, 15091609l, 164157963l, 15091656l, 43026009l, 43026010l, 15091678l, 19826860l, + 19826861l, 164157965l, 19973630l, 22277500l, 22277501l, 22277790l, 22277845l, 22277849l, + 22278089l, 22278090l, 22278109l, 22278111l, 22278116l, 22278120l, 22278128l, 22278626l, + 22328765l, 22328772l, 22328778l, 22328779l, 22330396l, 22679421l, 23040738l, 23040842l, + 23040846l, 23040858l, 23117246l, 23294746l, 23506818l, 23506829l, 23517268l, 23517271l, + 157868707l, 23598320l, 23598322l, 23598454l, 23737552l, 23737553l, 23737667l, + 145738805l, 24541552l, 24541863l, 24541865l, 25681100l, 25681103l, 25681278l, 25681279l, + 25681280l, 25681403l, 25681444l, 25681445l, 25690338l, 25690339l, 25690340l, 25690749l, + 43023896l, 43023897l, 25690753l, 25690754l, 25690755l, 25690809l, 25690810l, 38328051l, + 38328052l, 25691524l, 101712789l, 101712803l, 25692910l, 25698340l, 25698341l, + 25698342l, 25698343l, 25698344l, 25698346l, 25698348l, 25698352l, 25698353l, 25698354l, + 25698355l, 26263982l, 26955717l, 28366733l, 33971858l, 33971859l, 145738820l, 33995569l, + 23737666l, 41093668l, 44051303l, 44131233l, 44131234l, 25690748l, 25690752l, 44033101l, + 44202299l, 44317183l, 44317184l, 44834939l, 45317529l, 45675161l, 4275500l, 53574779l, + 55041560l, 62941752l, 70914841l, 100443831l, 103058540l, 103058539l, 25691035l, + 145809031l, 140969180l, 140969209l, 10989259l, 141570176l, 157621618l, 157868715l, + 157868706l, 157868717l, 157868710l, 157868709l, 157868714l, 157868705l, 157868718l, + 157868711l), + List.of("name", "NCN National Route 61", "network", "ncn", "ref", "61", "route", "bicycle", + "type", "route")), + new ExpectedRelation(267403, List.of(RelationMemberType.NODE, RelationMemberType.NODE), + List.of(502550970l, 502552090l), + List.of("name", "Oaktree Close", "naptan:StopAreaCode", "210G896", "naptan:StopAreaType", + "GPBS", "naptan:verified", "no", "site", "stop_area", "source", "naptan_import", "type", + "site")), + new ExpectedRelation(267404, List.of(RelationMemberType.NODE, RelationMemberType.NODE), + List.of(502550921l, 502552074l), + List.of("name", "Burfield Close", "naptan:StopAreaCode", "210G897", "naptan:StopAreaType", + "GPBS", "naptan:verified", "no", "site", "stop_area", "source", "naptan_import", "type", + "site")), + new ExpectedRelation(267400, List.of(RelationMemberType.NODE, RelationMemberType.NODE), + List.of(502550963l, 502552081l), + List.of("name", "Jasmine Gardens", "naptan:StopAreaCode", "210G895", "naptan:StopAreaType", + "GPBS", "naptan:verified", "no", "site", "stop_area", "source", "naptan_import", "type", + "site")), + }; + + URI testFileURL = RelationTest.class.getClassLoader().getResource("protobuf-test.osm.pbf").toURI(); + + Iterator<Entity> entities = new BlobSpliterator( + new RandomAccessFile(new File(testFileURL), "r"), + System.out::println) + .stream() + .flatMap(List::stream) + .filter(Relation.class::isInstance) + .iterator(); + + int expectedIndex = 0; + while (entities.hasNext() && expectedIndex < expected.length) { + Relation relation = (Relation) entities.next(); + ExpectedRelation ex = expected[expectedIndex]; + + Assert.assertEquals(ex.id, relation.getID()); + Assert.assertEquals(ex.tagMap(), relation.getTags()); + Assert.assertEquals("amount of members", ex.memberIDs.size(), relation.getMembers().size()); + + for (int i = 0; i < ex.memberIDs.size(); i++) { + Assert.assertEquals((long) ex.memberIDs.get(i), relation.getMembers().get(i).getID()); + Assert.assertEquals(ex.memberTypes.get(i), relation.getMembers().get(i).getType()); + } + + expectedIndex++; + } + } +} diff --git a/osm-protobuf/src/test/java/osm/protobuf/WayTest.java b/osm-protobuf/src/test/java/osm/protobuf/WayTest.java @@ -0,0 +1,286 @@ +package osm.protobuf; + +import java.io.File; +import java.io.RandomAccessFile; +import java.net.URI; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import osm.message.Entity; +import osm.message.Way; + +/** + * Demonstrates how to read a file. Reads sample.pbf from the resources folder + * and prints details about it to the standard output. + * + * @author Michael Tandy + */ +public class WayTest { + public static record ExpectedWay(long id, List<Long> children, List<String> tags) { + + public Map<String, String> tagMap() { + Map<String, String> map = new HashMap<>(); + for (int i = 0; i < tags.size(); i += 2) { + map.put(tags.get(i), tags.get(i + 1)); + } + return map; + } + } + + @Test + public void test() throws Exception { + ExpectedWay[] expected = new ExpectedWay[] { + new ExpectedWay(158788812, + List.of(1709246789l, 1709246746l, 1709246741l, 1709246791l), + List.of("highway", "footway")), + new ExpectedWay(53588781, + List.of(676945323l, 676945327l, 676945325l, 676945326l, 676945331l, + 676945323l), + List.of("landuse", "garages", "source", "survey")), + new ExpectedWay(158788810, + List.of(1709246675l, 1709246737l, 1709246734l, 1709246676l), + List.of("highway", "footway")), + new ExpectedWay(156255508, + List.of(45169425l, 1685167371l, 1685167341l, 1685167287l, 1685167296l, + 1685167315l, 1685167387l, + 1685167328l, 1685167373l, 1685167290l, 1685167282l, + 1685167313l, 1685167381l, + 1685167391l, 1685167394l, 1685167304l, 1685167376l, + 1606957353l), + List.of("carriageway_ref", "A", "highway", "motorway", "lanes", "3", + "layer", "-1", "lit", + "yes", "maxspeed", "national", "name", + "Hatfield Tunnel", "oneway", "yes", "ref", + "A1(M)", "source:maxspeed", "local_knowledge", "tunnel", + "yes")), + new ExpectedWay(54932035, + List.of(691202854l, 691202855l, 691202857l), + List.of("highway", "residential", "name", "Jasmine Gardens", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(16946553, + List.of(175697671l, 175697862l, 175697821l, 175697824l), + List.of("highway", "residential", "name", "Oak Tree Close")), + new ExpectedWay(52083876, + List.of(663806653l, 663806656l, 663806658l), + List.of("highway", "service")), + new ExpectedWay(55081202, + List.of(692944951l, 692944957l, 692944963l, 692944966l, 692944951l), + List.of("leisure", "common", "source", "yahoo")), + new ExpectedWay(16946600, + List.of(175698430l, 175698550l, 691203053l, 691203054l, 175698551l, + 1709246675l, 175698553l, + 1709246789l, 691203055l), + List.of("highway", "residential", "name", "Harmony Close", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(16946559, + List.of(175697862l, 175697881l), + List.of("created_by", "Potlatch 0.5d", "highway", "residential", "name", + "Oak Tree Close")), + new ExpectedWay(16945846, + List.of(175684459l, 175685100l, 175685102l, 175685104l, 676945189l, + 175685106l, 676945315l, + 175685109l, 175685111l, 175684462l), + List.of("highway", "residential", "name", "Stockbreach Close")), + new ExpectedWay(54932037, + List.of(175698553l, 691202860l), + List.of("highway", "residential", "name", "Harmony Close", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(16946584, + List.of(175698155l, 175698323l, 175698430l, 1709246676l, 175698324l, + 691203051l), + List.of("highway", "residential", "name", "The Minims", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(8361329, + List.of(25365926l, 25365927l, 25365928l), + List.of("highway", "residential", "name", "The Paddock", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(16945926, + List.of(175686498l, 175686201l, 663806661l, 175686499l), + List.of("highway", "residential", "name", "Wellfield Close")), + new ExpectedWay(52083878, + List.of(663806664l, 663806666l, 663806668l, 663806670l, 663806672l, + 663806673l, 663806664l), + List.of("leisure", "common", "source", "yahoo")), + new ExpectedWay(3084923, + List.of(675146l, 1145410964l, 1539682123l, 32950368l, 241806356l, + 675148l, 675149l, 820969139l, + 175698155l, 675150l, 390911769l, 647224486l, 675151l, + 175685910l, 14713407l, 175684463l, + 647057820l, 647224485l, 1739780291l), + List.of("abutters", "residential", "highway", "secondary", "name", + "Wellfield Road", "ref", + "B197")), + new ExpectedWay(8361331, + List.of(25365925l, 691203049l, 25365930l, 25365931l), + List.of("highway", "residential", "name", "Walsingham Close", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(54932038, + List.of(175699187l, 691202861l), + List.of("highway", "residential", "name", "Middlefield", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(16946620, + List.of(175698975l, 691203109l, 175699187l, 691203110l, 691203111l), + List.of("highway", "residential", "name", "Middlefield", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(157868709, + List.of(1701110757l, 1701110775l), + List.of("bridge", "yes", "cycleway", "track", "highway", "cycleway", + "layer", "1", "name", + "Alban Way", "ncn_ref", "61", "railway", "abandoned")), + new ExpectedWay(52083877, + List.of(663806656l, 663806661l), + List.of("highway", "service")), + new ExpectedWay(53588764, + List.of(676945267l, 677438877l, 677439947l, 676945292l, 676945293l, + 676945332l, 647224613l, + 175686498l), + List.of("highway", "footway")), + new ExpectedWay(49161822, + List.of(30983851l, 623624257l, 623624154l, 623624259l, 623624261l), + List.of("highway", "residential", "name", "Worcester Road")), + new ExpectedWay(49161823, + List.of(623624259l, 691203098l, 691203099l, 623624267l), + List.of("highway", "residential", "name", "Ely Close")), + new ExpectedWay(53588782, + List.of(676945327l, 676945329l), + List.of("bicycle", "no", "highway", "footway", "source", "survey")), + new ExpectedWay(54932044, + List.of(691202869l, 691202855l), + List.of("highway", "residential", "name", "Jasmine Gardens", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(49161817, + List.of(623624154l, 623624155l, 623624156l), + List.of("highway", "residential", "name", "Malvern Close")), + new ExpectedWay(53588780, + List.of(676945350l, 676945352l, 676945334l, 676945335l, 676945346l, + 676945347l, 676945350l), + List.of("building", "yes", "name", "Friendship House", "source", + "survey")), + new ExpectedWay(53588749, + List.of(676945199l, 676945316l, 676945317l, 676945195l, 676945319l, + 676945197l, 676945199l), + List.of("landuse", "garages", "source", "survey")), + new ExpectedWay(4673618, + List.of(675148l, 25365921l, 1539682089l, 25365922l, 1539682039l, + 691202838l, 25365923l, + 25365924l, 25365925l, 25365926l, 175698975l, 30983851l, + 175697671l, 30983852l, + 691203106l, 30983853l, 240134267l, 267826070l, + 365548881l, 32953193l, 175683944l, + 927070648l, 647225601l, 677440300l, 32953194l, + 673784380l, 32953195l), + List.of("highway", "tertiary", "name", "Lemsford Road")), + new ExpectedWay(53638158, + List.of(677439941l, 677439943l, 677439944l, 677439946l, 676945241l, + 175683342l, 653970876l, + 653970877l, 1697422651l, 651594517l, 676945267l, + 677438877l, 677439947l, 647224485l, + 1739780291l, 1739780285l, 1739780280l, 672628083l, + 651652534l, 651652536l, 1739780294l, + 623540483l, 623540479l, 623540472l, 623540467l, + 647224465l, 647057820l, 175684463l, + 14713407l, 175685910l, 675151l, 692887101l, 647224486l, + 647105115l, 647105117l, + 647105119l, 647105121l, 647105123l, 647105125l, + 647105127l, 647105128l, 647105129l, + 647105130l, 647105131l, 647105132l, 1111758071l, + 647105133l, 1111758069l, 647105134l, + 1111758072l, 647105135l, 1111758067l, 647105136l, + 647105137l, 647105138l, 647105139l, + 647105140l, 647105141l, 647105047l, 1692947499l, + 647105142l, 647105143l, 647105144l, + 647105145l, 647105146l, 647105147l, 647105148l, + 647105149l, 647105150l, 647105151l, + 647105152l, 647105153l, 647105154l, 647105155l, + 647105156l, 647105157l, 647105158l, + 647105159l, 647105160l, 647105161l, 647105162l, + 647105163l, 647105164l, 647105165l, + 647105166l, 534874147l, 534873285l, 534873274l, + 534873262l, 534873251l, 534873208l, + 534873185l, 534873171l, 647105167l, 647105168l, + 647105169l, 647105170l, 647105171l, + 647105172l, 647105102l, 647225601l, 677439941l), + List.of("landuse", "residential")), + new ExpectedWay(53588748, + List.of(676945322l, 676945325l, 676945327l, 676945320l, 676945192l, + 676945315l), + List.of("bicycle", "no", "highway", "footway", "source", "survey")), + new ExpectedWay(50772651, + List.of(175685111l, 676945322l, 175685506l, 647224613l), + List.of("highway", "residential", "name", "Town Fields")), + new ExpectedWay(158788824, + List.of(175685507l, 1709246749l), + List.of("highway", "footway")), + new ExpectedWay(54932042, + List.of(691202866l, 691202871l, 691202873l), + List.of("highway", "residential", "name", "Jasmine Gardens", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(53152061, + List.of(672663467l, 672663468l, 672663469l, 672663470l, 672663471l, + 672663473l, 672663474l, + 672663476l, 672663477l, 672663478l, 672663467l), + List.of("amenity", "retirement _home", "building", "yes", "name", + "Greenacres", "source:area", + "yahoo", "source:name", "survey")), + new ExpectedWay(53638215, + List.of(175685506l, 676945329l, 175685507l), + List.of("highway", "service", "source", "survey")), + new ExpectedWay(157868710, + List.of(1701110775l, 287659881l, 692887118l, 1709246791l, 1709246749l, + 240134269l, 240134268l), + List.of("cycleway", "track", "highway", "cycleway", "name", "Alban Way", + "ncn_ref", "61", + "railway", "abandoned")), + new ExpectedWay(54932036, + List.of(25365927l, 691202858l), + List.of("highway", "residential", "name", "The Paddock", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(54932039, + List.of(175697821l, 691202863l), + List.of("highway", "residential", "name", "Oak Tree Close", "source", + "OS_OpenData_StreetView")), + new ExpectedWay(55081204, + List.of(692945016l, 692945017l, 692945018l, 692945019l, 692945020l, + 692945021l, 692945022l, + 692945016l), + List.of("leisure", "common")), + new ExpectedWay(55071941, + List.of(692887118l, 692887101l), + List.of("foot", "yes", "highway", "footway")), + new ExpectedWay(157868707, + List.of(240134268l, 240134269l, 1709246749l, 1709246791l, 692887118l, + 287659881l), + List.of("cycleway", "track", "highway", "cycleway", "name", "Alban Way", + "ncn_ref", "61")), + + }; + + URI testFileURL = WayTest.class.getClassLoader().getResource("protobuf-test.osm.pbf").toURI(); + + Iterator<Entity> entities = new BlobSpliterator( + new RandomAccessFile(new File(testFileURL), "r"), + System.out::println) + .stream() + .flatMap(List::stream) + .filter(Way.class::isInstance) + .iterator(); + + int expectedIndex = 0; + while (entities.hasNext() && expectedIndex < expected.length) { + Way way = (Way) entities.next(); + ExpectedWay ex = expected[expectedIndex]; + + Assert.assertEquals(ex.id, way.getID()); + Assert.assertEquals(ex.children, way.children); + System.out.printf("expecting:\t%s\ngot:\t\t%s\n", ex.tagMap(), way.getTags()); + Assert.assertEquals(ex.tagMap(), way.getTags()); + + expectedIndex++; + } + } +} +\ No newline at end of file diff --git a/osm-protobuf/src/test/java/osm/protobuf/common/CountingCollector.java b/osm-protobuf/src/test/java/osm/protobuf/common/CountingCollector.java @@ -0,0 +1,58 @@ +package osm.protobuf.common; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; + +public class CountingCollector + implements Collector<Class<?>, Stack<Entry<Class<?>, AtomicInteger>>, List<Entry<Class<?>, Integer>>> { + @Override + public Supplier<Stack<Entry<Class<?>, AtomicInteger>>> supplier() { + return Stack::new; + } + + @Override + public BiConsumer<Stack<Entry<Class<?>, AtomicInteger>>, Class<?>> accumulator() { + return (stack, element) -> { + if (stack.isEmpty() || !stack.peek().getKey().equals(element)) { + stack.add(Map.entry(element, new AtomicInteger(1))); + } else { + stack.peek().getValue().incrementAndGet(); + } + }; + } + + @Override + public BinaryOperator<Stack<Entry<Class<?>, AtomicInteger>>> combiner() { + return (left, right) -> { + left.addAll(right); + return left; + }; + } + + @Override + public Function<Stack<Entry<Class<?>, AtomicInteger>>, List<Entry<Class<?>, Integer>>> finisher() { + return stack -> { + List<Entry<Class<?>, Integer>> list = new ArrayList<>(); + + for (Entry<Class<?>, AtomicInteger> entry : stack) + list.add(Map.entry(entry.getKey(), entry.getValue().get())); + + return list; + }; + } + + @Override + public Set<Characteristics> characteristics() { + return Set.of(); + } +} diff --git a/osm-protobuf/src/test/resources/protobuf-test.osm.pbf b/osm-protobuf/src/test/resources/protobuf-test.osm.pbf Binary files differ. diff --git a/protobuf-native/src/main/java/protobuf/Message.java b/protobuf-native/src/main/java/protobuf/Message.java @@ -1,43 +0,0 @@ -package protobuf; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.util.Iterator; - -/** - * Functional interface representing a message parser for Protobuf data. - * - * @param <T> The type of the parsed message. - */ -@FunctionalInterface -public interface Message<T> { - - /** - * Parses a byte array as a message. - * - * @param input The byte array containing Protobuf data. - * @return The parsed message. - */ - default T parse(byte[] input) { - return parse(new ByteArrayInputStream(input), input.length); - } - - /** - * Parses an input stream with a specified length as a message. - * - * @param input The input stream containing Protobuf data. - * @param length The length of the Protobuf data in the input stream. - * @return The parsed message. - */ - default T parse(InputStream input, int length) { - return parse(new MessageIterator(input, length)); - } - - /** - * Parses Protobuf data using a custom iterator. - * - * @param tags The iterator providing Protobuf data elements. - * @return The parsed message. - */ - T parse(Iterator<ProtobufReader> tags); -} diff --git a/protobuf-native/src/main/java/protobuf/MessageDecoder.java b/protobuf-native/src/main/java/protobuf/MessageDecoder.java @@ -0,0 +1,43 @@ +package protobuf; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Iterator; + +/** + * Functional interface representing a message parser for Protobuf data. + * + * @param <T> The type of the parsed message. + */ +@FunctionalInterface +public interface MessageDecoder<T> { + + /** + * Parses a byte array as a message. + * + * @param input The byte array containing Protobuf data. + * @return The parsed message. + */ + default T parse(byte[] input) { + return parse(new ByteArrayInputStream(input), input.length); + } + + /** + * Parses an input stream with a specified length as a message. + * + * @param input The input stream containing Protobuf data. + * @param length The length of the Protobuf data in the input stream. + * @return The parsed message. + */ + default T parse(InputStream input, int length) { + return parse(new MessageIterator(input, length)); + } + + /** + * Parses Protobuf data using a custom iterator. + * + * @param tags The iterator providing Protobuf data elements. + * @return The parsed message. + */ + T parse(Iterator<ProtobufReader> tags); +} diff --git a/protobuf-native/src/main/java/protobuf/MessageEncoder.java b/protobuf-native/src/main/java/protobuf/MessageEncoder.java @@ -0,0 +1,43 @@ +package protobuf; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Iterator; + +/** + * Functional interface representing a message parser for Protobuf data. + * + * @param <T> The type of the parsed message. + */ +@FunctionalInterface +public interface MessageEncoder<T> { + + /** + * Parses a byte array as a message. + * + * @param input The byte array containing Protobuf data. + * @return The parsed message. + */ + default T parse(byte[] input) { + return parse(new ByteArrayInputStream(input), input.length); + } + + /** + * Parses an input stream with a specified length as a message. + * + * @param input The input stream containing Protobuf data. + * @param length The length of the Protobuf data in the input stream. + * @return The parsed message. + */ + default T parse(InputStream input, int length) { + return parse(new MessageIterator(input, length)); + } + + /** + * Parses Protobuf data using a custom iterator. + * + * @param tags The iterator providing Protobuf data elements. + * @return The parsed message. + */ + T parse(Iterator<ProtobufReader> tags); +} diff --git a/protobuf-native/src/main/java/protobuf/MessageIterator.java b/protobuf-native/src/main/java/protobuf/MessageIterator.java @@ -103,6 +103,6 @@ public class MessageIterator implements Iterator<ProtobufReader> { // least-significant 3 bits are the type, remaining is the tag value // type = tag & 0b00000111 - return new SimpleProtobufReader(this, WireType.values()[tag & 0x07], tag >> 3); + return new ProtobufReader(this, WireType.values()[tag & 0x07], tag >> 3); } } \ No newline at end of file diff --git a/protobuf-native/src/main/java/protobuf/ProtobufReader.java b/protobuf-native/src/main/java/protobuf/ProtobufReader.java @@ -1,6 +1,8 @@ package protobuf; +import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.function.BinaryOperator; import java.util.function.Consumer; @@ -8,114 +10,322 @@ import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; +import protobuf.exception.InputException; +import protobuf.exception.OverflowException; +import protobuf.exception.UnexpectedTagException; +import protobuf.exception.WireTypeException; + /** * Represents an interface for parsing Protobuf wire elements, providing methods * for reading various Protobuf data types. */ -public interface ProtobufReader { +public class ProtobufReader { + private final MessageIterator message; + private final WireType type; + private final int tag; + private boolean resetType = false; + + public ProtobufReader(MessageIterator message, WireType type, int tag) { + this.message = message; + this.type = type; + this.tag = tag; + } /** * Gets the underlying input stream. * * @return The input stream. */ - InputStream getInputStream(); + public InputStream getInputStream() { + return message.input; + } /** * Gets the Protobuf wire type. * * @return The wire type. */ - WireType getType(); + public WireType getType() { + return resetType ? null : type; + } /** * Resets the Protobuf wire type. * Useful for parsing packed streams. */ - void resetType(); + public void resetType() { + resetType = true; + } /** * Gets the tag associated with the wire element. * * @return The wire tag. */ - int tag(); + public int tag() { + return tag; + } /** * Reads a signed variable-length integer as a 64-bit integer. * * @return The parsed signed 64-bit integer. */ - long svarint64(); + public long svarint64() { + long n = varint64(); + return (n & 0x01) == 0 + ? (n >> 1) + : -(n >> 1) - 1; + } /** * Parses a variable-length integer as a signed 64-bit integer. * * @return The parsed signed 64-bit integer. */ - long varint64(); + public long varint64() { + return varint64(false); + } /** * Reads a signed variable-length integer as a 32-bit integer. * * @return The parsed signed 32-bit integer. */ - long svarint32(); + public long varint64(boolean ignoreType) { + if (!ignoreType && getType() != null && getType() != WireType.VARINT) + throw new WireTypeException(WireType.VARINT, getType()); + + long result = 0; + long b = 0; + int shift = 0; + while (shift < 64 && message.length > 0) { + try { + b = message.input.read(); + } catch (IOException exc) { + throw new InputException(exc); + } + if (b == -1) + break; + + message.length--; + + result |= (b & 0x7f) << shift; + shift += 7; + if ((b & 0x80) == 0) + return result; + } + + throw new OverflowException("input exceed"); + } + + public long svarint32() { + int n = varint32(); + return (n & 0x01) == 0 + ? (n >> 1) + : -(n >> 1); + + } /** * Parses a variable-length integer as a signed 32-bit integer. * * @return The parsed signed 32-bit integer. */ - int varint32(); + public int varint32() { + return varint32(false); + } + + private int varint32(boolean ignoreType) { + if (!ignoreType && getType() != null && getType() != WireType.VARINT) + throw new WireTypeException(WireType.VARINT, getType()); + + int result = 0; + int b = 0; + int shift = 0; + while (shift < 32 && message.length > 0) { + try { + b = message.input.read(); + } catch (IOException exc) { + throw new InputException(exc); + } + if (b == -1) + break; + + message.length--; + + result |= (b & 0x7f) << shift; + shift += 7; + if ((b & 0x80) == 0) + return result; + } + throw new OverflowException("input exceed"); + } /** * Skips the variable-length integer. */ - void skipVarint(); + public void skipVarint() { + if (getType() != null && getType() != WireType.VARINT) + throw new WireTypeException(WireType.VARINT, getType()); + + int b = 0; + while (message.length > 0) { + try { + b = message.input.read(); + } catch (IOException exc) { + throw new InputException(exc); + } + if (b == -1) + break; + + message.length--; + if ((b & 0x80) == 0) + return; + } + throw new OverflowException("input exceed"); + } /** * Reads a fixed 64-bit integer. * * @return The parsed 64-bit integer. */ - long fixed64(); + public long fixed64() { + if (getType() != null && getType() != WireType.I64) + throw new WireTypeException(WireType.I64, getType()); + + if (message.length < 8) + throw new OverflowException("input exceed"); + + byte[] bytes; + try { + bytes = message.input.readNBytes(8); + } catch (IOException exc) { + throw new InputException(exc); + } + long result = 0; + + for (int i = bytes.length - 1; i >= 0; i--) { + result <<= 8; + result |= bytes[i]; + } + + return result; + } /** * Skips a fixed 64-bit integer. */ - void skip64(); + public void skip64() { + if (getType() != null && getType() != WireType.I64) + throw new WireTypeException(WireType.I64, getType()); + + if (message.length < 8) + throw new OverflowException("input exceed"); + + message.length -= 8; + try { + message.input.skipNBytes(8); + } catch (IOException exc) { + throw new InputException(exc); + } + } /** * Reads a fixed 32-bit integer. * * @return The parsed 32-bit integer. */ - int fixed32(); + public int fixed32() { + if (getType() != null && getType() != WireType.I32) + throw new WireTypeException(WireType.I32, getType()); + + if (message.length < 4) + throw new OverflowException("input exceed"); + + byte[] bytes; + try { + bytes = message.input.readNBytes(4); + } catch (IOException exc) { + throw new InputException(exc); + } + int result = 0; + + for (int i = bytes.length - 1; i >= 0; i--) { + result <<= 8; + result |= bytes[i]; + } + + return result; + } /** * Skips a fixed 32-bit integer. */ - void skip32(); + public void skip32() { + if (getType() != null && getType() != WireType.I32) + throw new WireTypeException(WireType.I32, getType()); + + if (message.length < 4) + throw new OverflowException("input exceed"); + + message.length -= 4; + try { + message.input.skipNBytes(4); + } catch (IOException exc) { + throw new InputException(exc); + } + } /** * Reads a byte array. * * @return The read byte array. */ - byte[] bytes(); + public byte[] bytes() { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + message.length -= len; + try { + return message.input.readNBytes(len); + } catch (IOException exc) { + throw new InputException(exc); + } + } /** * Skips a byte array. */ - void skipBytes(); + public void skipBytes() { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + message.length -= len; + try { + message.input.skipNBytes(len); + } catch (IOException exc) { + throw new InputException(exc); + } + } /** * Reads a string. * * @return The read string. */ - String string(); + public String string() { + return new String(bytes(), StandardCharsets.UTF_8); + } /** * Reads a message using the provided handler. @@ -124,7 +334,18 @@ public interface ProtobufReader { * @param <T> The type of the parsed message. * @return The parsed message. */ - <T> T message(Message<T> handler); + public <T> T message(Message<T> handler) { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + message.length -= len; + + return handler.parse(message.input, len); + } /** * Reads a message using the provided handler and applies a mapping function to @@ -135,7 +356,10 @@ public interface ProtobufReader { * @param <T> The type of the parsed message. * @return The parsed message. */ - <T> T message(Message<T> handler, UnaryOperator<byte[]> map); + public <T> T message(Message<T> handler, UnaryOperator<byte[]> map) { + byte[] buffer = bytes(); + return handler.parse(map.apply(buffer)); + } /** * Creates an iterator for packed values. @@ -144,7 +368,9 @@ public interface ProtobufReader { * @param <T> The type of the scalar values. * @return The iterator for packed values. */ - <T> Iterator<T> packed(Supplier<T> scalar); + public <T> Iterator<T> packed(Supplier<T> scalar) { + return packed(scalar, v -> v); + } /** * Creates an iterator for packed values, applying a mapping function to each @@ -156,7 +382,30 @@ public interface ProtobufReader { * @param <M> The type of the mapped values. * @return The iterator for packed values. */ - <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map); + public <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map) { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + int end = message.length - len; + + resetType(); + + return new Iterator<>() { + public boolean hasNext() { + if (message.length < end) + throw new OverflowException("packed string overused"); + return message.length > end; + } + + public M next() { + return map.apply(scalar.get()); + } + }; + } /** * Creates an iterator for packed values with an initial value and a binary @@ -168,7 +417,9 @@ public interface ProtobufReader { * @param <T> The type of the scalar values. * @return The iterator for packed values. */ - <T> Iterator<T> packed(Supplier<T> scalar, T init, BinaryOperator<T> operator); + public <T> Iterator<T> packed(Supplier<T> scalar, T init, BinaryOperator<T> operator) { + return packed(scalar, v -> v, init, operator); + } /** * Creates an iterator for packed values with an initial value, a binary @@ -182,7 +433,32 @@ public interface ProtobufReader { * @param <M> The type of the mapped values. * @return The iterator for packed values. */ - <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map, M init, BinaryOperator<M> operator); + public <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map, M init, BinaryOperator<M> operator) { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + int end = message.length - len; + + resetType(); + + return new Iterator<>() { + M value = init; + + public boolean hasNext() { + if (message.length < end) + throw new OverflowException("packed string overused"); + return message.length > end; + } + + public M next() { + return value = operator.apply(value, map.apply(scalar.get())); + } + }; + } /** * Defers the execution of a consumer with a supplied value. @@ -191,7 +467,10 @@ public interface ProtobufReader { * @param defer The consumer to be deferred. * @param <T> The type of the supplied value. */ - <T> void delayed(Supplier<T> supplier, Consumer<T> defer); + public <T> void delayed(Supplier<T> supplier, Consumer<T> defer) { + T buffer = supplier.get(); + message.delayed.add(() -> defer.accept(buffer)); + } /** * Defers the execution of a consumer with a mapped supplied value. @@ -201,7 +480,10 @@ public interface ProtobufReader { * @param defer The consumer to be deferred. * @param <T> The type of the supplied value. */ - <T> void delayed(Supplier<T> supplier, UnaryOperator<T> map, Consumer<T> defer); + public <T> void delayed(Supplier<T> supplier, UnaryOperator<T> map, Consumer<T> defer) { + T buffer = supplier.get(); + message.delayed.add(() -> defer.accept(map.apply(buffer))); + } /** * Defers the execution of a consumer with a parsed message using the provided @@ -211,7 +493,10 @@ public interface ProtobufReader { * @param defer The consumer to be deferred. * @param <T> The type of the parsed message. */ - <T> void delayed(Message<T> handler, Consumer<T> defer); + public <T> void delayed(Message<T> handler, Consumer<T> defer) { + byte[] buffer = bytes(); + message.delayed.add(() -> defer.accept(handler.parse(buffer))); + } /** * Defers the execution of a consumer with a mapped parsed message using the @@ -222,15 +507,38 @@ public interface ProtobufReader { * @param defer The consumer to be deferred. * @param <T> The type of the parsed message. */ - <T> void delayed(Message<T> handler, UnaryOperator<byte[]> map, Consumer<T> defer); + public <T> void delayed(Message<T> handler, UnaryOperator<byte[]> map, Consumer<T> defer) { + byte[] buffer = bytes(); + message.delayed.add(() -> defer.accept(handler.parse(map.apply(buffer)))); + } /** * Skips the current wire element based on its type. */ - void skip(); + public void skip() { + switch (getType()) { + case VARINT: + skipVarint(); + break; + case I64: + skip64(); + break; + case LEN: + skipBytes(); + break; + case I32: + skip32(); + break; + case SGROUP: + case EGROUP: + throw new UnsupportedOperationException("cannot skip sgroup of egroup"); + } + } /** * Throws an {@link UnexpectedTagException} for the current tag. */ - void throwUnexpected(); + public void throwUnexpected() { + throw new UnexpectedTagException(tag); + } } diff --git a/protobuf-native/src/main/java/protobuf/ProtobufWriter.java b/protobuf-native/src/main/java/protobuf/ProtobufWriter.java @@ -0,0 +1,545 @@ +package protobuf; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.function.BinaryOperator; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +import protobuf.exception.InputException; +import protobuf.exception.OverflowException; +import protobuf.exception.UnexpectedTagException; +import protobuf.exception.WireTypeException; + +/** + * Represents an interface for parsing Protobuf wire elements, providing methods + * for reading various Protobuf data types. + */ +public class ProtobufWriter { + private final MessageIterator message; + private final WireType type; + private final int tag; + private boolean resetType = false; + + public ProtobufReader(MessageIterator message, WireType type, int tag) { + this.message = message; + this.type = type; + this.tag = tag; + } + + /** + * Gets the underlying input stream. + * + * @return The input stream. + */ + public InputStream getInputStream() { + return message.input; + } + + /** + * Gets the Protobuf wire type. + * + * @return The wire type. + */ + public WireType getType() { + return resetType ? null : type; + } + + /** + * Resets the Protobuf wire type. + * Useful for parsing packed streams. + */ + public void resetType() { + resetType = true; + } + + /** + * Gets the tag associated with the wire element. + * + * @return The wire tag. + */ + public int tag() { + return tag; + } + + /** + * Reads a signed variable-length integer as a 64-bit integer. + * + * @return The parsed signed 64-bit integer. + */ + public long svarint64() { + long n = varint64(); + return (n & 0x01) == 0 + ? (n >> 1) + : -(n >> 1) - 1; + } + + /** + * Parses a variable-length integer as a signed 64-bit integer. + * + * @return The parsed signed 64-bit integer. + */ + public long varint64() { + return varint64(false); + } + + /** + * Reads a signed variable-length integer as a 32-bit integer. + * + * @return The parsed signed 32-bit integer. + */ + public long varint64(boolean ignoreType) { + if (!ignoreType && getType() != null && getType() != WireType.VARINT) + throw new WireTypeException(WireType.VARINT, getType()); + + long result = 0; + long b = 0; + int shift = 0; + while (shift < 64 && message.length > 0) { + try { + b = message.input.read(); + } catch (IOException exc) { + throw new InputException(exc); + } + if (b == -1) + break; + + message.length--; + + result |= (b & 0x7f) << shift; + shift += 7; + if ((b & 0x80) == 0) + return result; + } + + throw new OverflowException("input exceed"); + } + + public long svarint32() { + int n = varint32(); + return (n & 0x01) == 0 + ? (n >> 1) + : -(n >> 1); + + } + + /** + * Parses a variable-length integer as a signed 32-bit integer. + * + * @return The parsed signed 32-bit integer. + */ + public int varint32() { + return varint32(false); + } + + private int varint32(boolean ignoreType) { + if (!ignoreType && getType() != null && getType() != WireType.VARINT) + throw new WireTypeException(WireType.VARINT, getType()); + + int result = 0; + int b = 0; + int shift = 0; + while (shift < 32 && message.length > 0) { + try { + b = message.input.read(); + } catch (IOException exc) { + throw new InputException(exc); + } + if (b == -1) + break; + + message.length--; + + result |= (b & 0x7f) << shift; + shift += 7; + if ((b & 0x80) == 0) + return result; + } + throw new OverflowException("input exceed"); + } + + /** + * Skips the variable-length integer. + */ + public void skipVarint() { + if (getType() != null && getType() != WireType.VARINT) + throw new WireTypeException(WireType.VARINT, getType()); + + int b = 0; + while (message.length > 0) { + try { + b = message.input.read(); + } catch (IOException exc) { + throw new InputException(exc); + } + if (b == -1) + break; + + message.length--; + if ((b & 0x80) == 0) + return; + } + throw new OverflowException("input exceed"); + } + + /** + * Reads a fixed 64-bit integer. + * + * @return The parsed 64-bit integer. + */ + public long fixed64() { + if (getType() != null && getType() != WireType.I64) + throw new WireTypeException(WireType.I64, getType()); + + if (message.length < 8) + throw new OverflowException("input exceed"); + + byte[] bytes; + try { + bytes = message.input.readNBytes(8); + } catch (IOException exc) { + throw new InputException(exc); + } + long result = 0; + + for (int i = bytes.length - 1; i >= 0; i--) { + result <<= 8; + result |= bytes[i]; + } + + return result; + } + + /** + * Skips a fixed 64-bit integer. + */ + public void skip64() { + if (getType() != null && getType() != WireType.I64) + throw new WireTypeException(WireType.I64, getType()); + + if (message.length < 8) + throw new OverflowException("input exceed"); + + message.length -= 8; + try { + message.input.skipNBytes(8); + } catch (IOException exc) { + throw new InputException(exc); + } + } + + /** + * Reads a fixed 32-bit integer. + * + * @return The parsed 32-bit integer. + */ + public int fixed32() { + if (getType() != null && getType() != WireType.I32) + throw new WireTypeException(WireType.I32, getType()); + + if (message.length < 4) + throw new OverflowException("input exceed"); + + byte[] bytes; + try { + bytes = message.input.readNBytes(4); + } catch (IOException exc) { + throw new InputException(exc); + } + int result = 0; + + for (int i = bytes.length - 1; i >= 0; i--) { + result <<= 8; + result |= bytes[i]; + } + + return result; + } + + /** + * Skips a fixed 32-bit integer. + */ + public void skip32() { + if (getType() != null && getType() != WireType.I32) + throw new WireTypeException(WireType.I32, getType()); + + if (message.length < 4) + throw new OverflowException("input exceed"); + + message.length -= 4; + try { + message.input.skipNBytes(4); + } catch (IOException exc) { + throw new InputException(exc); + } + } + + /** + * Reads a byte array. + * + * @return The read byte array. + */ + public byte[] bytes() { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + message.length -= len; + try { + return message.input.readNBytes(len); + } catch (IOException exc) { + throw new InputException(exc); + } + } + + /** + * Skips a byte array. + */ + public void skipBytes() { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + message.length -= len; + try { + message.input.skipNBytes(len); + } catch (IOException exc) { + throw new InputException(exc); + } + } + + /** + * Reads a string. + * + * @return The read string. + */ + public String string() { + return new String(bytes(), StandardCharsets.UTF_8); + } + + /** + * Reads a message using the provided handler. + * + * @param handler The message handler. + * @param <T> The type of the parsed message. + * @return The parsed message. + */ + public <T> T message(Message<T> handler) { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + message.length -= len; + + return handler.parse(message.input, len); + } + + /** + * Reads a message using the provided handler and applies a mapping function to + * the byte array. + * + * @param handler The message handler. + * @param map The mapping function for the byte array. + * @param <T> The type of the parsed message. + * @return The parsed message. + */ + public <T> T message(Message<T> handler, UnaryOperator<byte[]> map) { + byte[] buffer = bytes(); + return handler.parse(map.apply(buffer)); + } + + /** + * Creates an iterator for packed values. + * + * @param scalar The scalar supplier. + * @param <T> The type of the scalar values. + * @return The iterator for packed values. + */ + public <T> Iterator<T> packed(Supplier<T> scalar) { + return packed(scalar, v -> v); + } + + /** + * Creates an iterator for packed values, applying a mapping function to each + * scalar value. + * + * @param scalar The scalar supplier. + * @param map The mapping function for scalar values. + * @param <T> The type of the scalar values. + * @param <M> The type of the mapped values. + * @return The iterator for packed values. + */ + public <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map) { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + int end = message.length - len; + + resetType(); + + return new Iterator<>() { + public boolean hasNext() { + if (message.length < end) + throw new OverflowException("packed string overused"); + return message.length > end; + } + + public M next() { + return map.apply(scalar.get()); + } + }; + } + + /** + * Creates an iterator for packed values with an initial value and a binary + * operator. + * + * @param scalar The scalar supplier. + * @param init The initial value. + * @param operator The binary operator. + * @param <T> The type of the scalar values. + * @return The iterator for packed values. + */ + public <T> Iterator<T> packed(Supplier<T> scalar, T init, BinaryOperator<T> operator) { + return packed(scalar, v -> v, init, operator); + } + + /** + * Creates an iterator for packed values with an initial value, a binary + * operator, and a mapping function. + * + * @param scalar The scalar supplier. + * @param map The mapping function for scalar values. + * @param init The initial value. + * @param operator The binary operator. + * @param <T> The type of the scalar values. + * @param <M> The type of the mapped values. + * @return The iterator for packed values. + */ + public <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map, M init, BinaryOperator<M> operator) { + if (getType() != null && getType() != WireType.LEN) + throw new WireTypeException(WireType.LEN, getType()); + + int len = varint32(true); + if (message.length < len) + throw new OverflowException("input exceed"); + + int end = message.length - len; + + resetType(); + + return new Iterator<>() { + M value = init; + + public boolean hasNext() { + if (message.length < end) + throw new OverflowException("packed string overused"); + return message.length > end; + } + + public M next() { + return value = operator.apply(value, map.apply(scalar.get())); + } + }; + } + + /** + * Defers the execution of a consumer with a supplied value. + * + * @param supplier The value supplier. + * @param defer The consumer to be deferred. + * @param <T> The type of the supplied value. + */ + public <T> void delayed(Supplier<T> supplier, Consumer<T> defer) { + T buffer = supplier.get(); + message.delayed.add(() -> defer.accept(buffer)); + } + + /** + * Defers the execution of a consumer with a mapped supplied value. + * + * @param supplier The value supplier. + * @param map The mapping function for the supplied value. + * @param defer The consumer to be deferred. + * @param <T> The type of the supplied value. + */ + public <T> void delayed(Supplier<T> supplier, UnaryOperator<T> map, Consumer<T> defer) { + T buffer = supplier.get(); + message.delayed.add(() -> defer.accept(map.apply(buffer))); + } + + /** + * Defers the execution of a consumer with a parsed message using the provided + * handler. + * + * @param handler The message handler. + * @param defer The consumer to be deferred. + * @param <T> The type of the parsed message. + */ + public <T> void delayed(Message<T> handler, Consumer<T> defer) { + byte[] buffer = bytes(); + message.delayed.add(() -> defer.accept(handler.parse(buffer))); + } + + /** + * Defers the execution of a consumer with a mapped parsed message using the + * provided handler. + * + * @param handler The message handler. + * @param map The mapping function for the parsed message. + * @param defer The consumer to be deferred. + * @param <T> The type of the parsed message. + */ + public <T> void delayed(Message<T> handler, UnaryOperator<byte[]> map, Consumer<T> defer) { + byte[] buffer = bytes(); + message.delayed.add(() -> defer.accept(handler.parse(map.apply(buffer)))); + } + + /** + * Skips the current wire element based on its type. + */ + public void skip() { + switch (getType()) { + case VARINT: + skipVarint(); + break; + case I64: + skip64(); + break; + case LEN: + skipBytes(); + break; + case I32: + skip32(); + break; + case SGROUP: + case EGROUP: + throw new UnsupportedOperationException("cannot skip sgroup of egroup"); + } + } + + /** + * Throws an {@link UnexpectedTagException} for the current tag. + */ + public void throwUnexpected() { + throw new UnexpectedTagException(tag); + } +} +} +\ No newline at end of file diff --git a/protobuf-native/src/main/java/protobuf/SimpleProtobufReader.java b/protobuf-native/src/main/java/protobuf/SimpleProtobufReader.java @@ -1,409 +0,0 @@ -package protobuf; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.function.BinaryOperator; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; - -import protobuf.exception.InputException; -import protobuf.exception.OverflowException; -import protobuf.exception.UnexpectedTagException; -import protobuf.exception.WireTypeException; - -class SimpleProtobufReader implements ProtobufReader { - private final MessageIterator message; - private final WireType type; - private final int tag; - private boolean resetType = false; - - public SimpleProtobufReader(MessageIterator message, WireType type, int tag) { - this.message = message; - this.type = type; - this.tag = tag; - } - - @Override - public InputStream getInputStream() { - return message.input; - } - - @Override - public WireType getType() { - return resetType ? null : type; - } - - @Override - public void resetType() { - resetType = true; - } - - @Override - public int tag() { - return tag; - } - - @Override - public long svarint64() { - long n = varint64(); - return (n & 0x01) == 0 - ? (n >> 1) - : -(n >> 1) - 1; - } - - @Override - public long varint64() { - return varint64(false); - } - - public long varint64(boolean ignoreType) { - if (!ignoreType && getType() != null && getType() != WireType.VARINT) - throw new WireTypeException(WireType.VARINT, getType()); - - long result = 0; - long b = 0; - int shift = 0; - while (shift < 64 && message.length > 0) { - try { - b = message.input.read(); - } catch (IOException exc) { - throw new InputException(exc); - } - if (b == -1) - break; - - message.length--; - - result |= (b & 0x7f) << shift; - shift += 7; - if ((b & 0x80) == 0) - return result; - } - - throw new OverflowException("input exceed"); - } - - @Override - public long svarint32() { - int n = varint32(); - return (n & 0x01) == 0 - ? (n >> 1) - : -(n >> 1); - - } - - @Override - public int varint32() { - return varint32(false); - } - - public int varint32(boolean ignoreType) { - if (!ignoreType && getType() != null && getType() != WireType.VARINT) - throw new WireTypeException(WireType.VARINT, getType()); - - int result = 0; - int b = 0; - int shift = 0; - while (shift < 32 && message.length > 0) { - try { - b = message.input.read(); - } catch (IOException exc) { - throw new InputException(exc); - } - if (b == -1) - break; - - message.length--; - - result |= (b & 0x7f) << shift; - shift += 7; - if ((b & 0x80) == 0) - return result; - } - throw new OverflowException("input exceed"); - } - - @Override - public void skipVarint() { - if (getType() != null && getType() != WireType.VARINT) - throw new WireTypeException(WireType.VARINT, getType()); - - int b = 0; - while (message.length > 0) { - try { - b = message.input.read(); - } catch (IOException exc) { - throw new InputException(exc); - } - if (b == -1) - break; - - message.length--; - if ((b & 0x80) == 0) - return; - } - throw new OverflowException("input exceed"); - } - - @Override - public long fixed64() { - if (getType() != null && getType() != WireType.I64) - throw new WireTypeException(WireType.I64, getType()); - - if (message.length < 8) - throw new OverflowException("input exceed"); - - byte[] bytes; - try { - bytes = message.input.readNBytes(8); - } catch (IOException exc) { - throw new InputException(exc); - } - long result = 0; - - for (int i = bytes.length - 1; i >= 0; i--) { - result <<= 8; - result |= bytes[i]; - } - - return result; - } - - @Override - public void skip64() { - if (getType() != null && getType() != WireType.I64) - throw new WireTypeException(WireType.I64, getType()); - - if (message.length < 8) - throw new OverflowException("input exceed"); - - message.length -= 8; - try { - message.input.skipNBytes(8); - } catch (IOException exc) { - throw new InputException(exc); - } - } - - @Override - public int fixed32() { - if (getType() != null && getType() != WireType.I32) - throw new WireTypeException(WireType.I32, getType()); - - if (message.length < 4) - throw new OverflowException("input exceed"); - - byte[] bytes; - try { - bytes = message.input.readNBytes(4); - } catch (IOException exc) { - throw new InputException(exc); - } - int result = 0; - - for (int i = bytes.length - 1; i >= 0; i--) { - result <<= 8; - result |= bytes[i]; - } - - return result; - } - - @Override - public void skip32() { - if (getType() != null && getType() != WireType.I32) - throw new WireTypeException(WireType.I32, getType()); - - if (message.length < 4) - throw new OverflowException("input exceed"); - - message.length -= 4; - try { - message.input.skipNBytes(4); - } catch (IOException exc) { - throw new InputException(exc); - } - } - - @Override - public byte[] bytes() { - if (getType() != null && getType() != WireType.LEN) - throw new WireTypeException(WireType.LEN, getType()); - - int len = varint32(true); - if (message.length < len) - throw new OverflowException("input exceed"); - - message.length -= len; - try { - return message.input.readNBytes(len); - } catch (IOException exc) { - throw new InputException(exc); - } - } - - @Override - public void skipBytes() { - if (getType() != null && getType() != WireType.LEN) - throw new WireTypeException(WireType.LEN, getType()); - - int len = varint32(true); - if (message.length < len) - throw new OverflowException("input exceed"); - - message.length -= len; - try { - message.input.skipNBytes(len); - } catch (IOException exc) { - throw new InputException(exc); - } - } - - @Override - public String string() { - return new String(bytes(), StandardCharsets.UTF_8); - } - - @Override - public <T> T message(Message<T> handler) { - if (getType() != null && getType() != WireType.LEN) - throw new WireTypeException(WireType.LEN, getType()); - - int len = varint32(true); - if (message.length < len) - throw new OverflowException("input exceed"); - - message.length -= len; - - return handler.parse(message.input, len); - } - - @Override - public <T> T message(Message<T> handler, UnaryOperator<byte[]> map) { - byte[] buffer = bytes(); - return handler.parse(map.apply(buffer)); - } - - @Override - public <T> Iterator<T> packed(Supplier<T> scalar) { - return packed(scalar, v -> v); - } - - @Override - public <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map) { - if (getType() != null && getType() != WireType.LEN) - throw new WireTypeException(WireType.LEN, getType()); - - int len = varint32(true); - if (message.length < len) - throw new OverflowException("input exceed"); - - int end = message.length - len; - - resetType(); - - return new Iterator<>() { - @Override - public boolean hasNext() { - if (message.length < end) - throw new OverflowException("packed string overused"); - return message.length > end; - } - - @Override - public M next() { - return map.apply(scalar.get()); - } - }; - } - - @Override - public <T> Iterator<T> packed(Supplier<T> scalar, T init, BinaryOperator<T> operator) { - return packed(scalar, v -> v, init, operator); - } - - @Override - public <T, M> Iterator<M> packed(Supplier<T> scalar, Function<T, M> map, M init, BinaryOperator<M> operator) { - if (getType() != null && getType() != WireType.LEN) - throw new WireTypeException(WireType.LEN, getType()); - - int len = varint32(true); - if (message.length < len) - throw new OverflowException("input exceed"); - - int end = message.length - len; - - resetType(); - - return new Iterator<>() { - M value = init; - - @Override - public boolean hasNext() { - if (message.length < end) - throw new OverflowException("packed string overused"); - return message.length > end; - } - - @Override - public M next() { - return value = operator.apply(value, map.apply(scalar.get())); - } - }; - } - - @Override - public <T> void delayed(Supplier<T> supplier, Consumer<T> defer) { - T buffer = supplier.get(); - message.delayed.add(() -> defer.accept(buffer)); - } - - @Override - public <T> void delayed(Supplier<T> supplier, UnaryOperator<T> map, Consumer<T> defer) { - T buffer = supplier.get(); - message.delayed.add(() -> defer.accept(map.apply(buffer))); - } - - /// make sure that parameters are ready, thus execute it after loop - @Override - public <T> void delayed(Message<T> handler, Consumer<T> defer) { - byte[] buffer = bytes(); - message.delayed.add(() -> defer.accept(handler.parse(buffer))); - } - - @Override - public <T> void delayed(Message<T> handler, UnaryOperator<byte[]> map, Consumer<T> defer) { - byte[] buffer = bytes(); - message.delayed.add(() -> defer.accept(handler.parse(map.apply(buffer)))); - } - - @Override - public void skip() { - switch (getType()) { - case VARINT: - skipVarint(); - break; - case I64: - skip64(); - break; - case LEN: - skipBytes(); - break; - case I32: - skip32(); - break; - case SGROUP: - case EGROUP: - throw new UnsupportedOperationException("cannot skip sgroup of egroup"); - } - } - - @Override - public void throwUnexpected() { - throw new UnexpectedTagException(tag); - } -} diff --git a/test.py b/test.py @@ -0,0 +1,303 @@ +relationxml = ''' +<osm> + <relation id="21855" version="7" timestamp="2012-03-21T21:37:54Z" uid="14020" user="Nick Austin" changeset="11057350"> + <member type="way" ref="156255508" role=""/> + <member type="way" ref="156255507" role=""/> + <tag k="name" v="Hatfield Tunnel"/> + <tag k="type" v="tunnel"/> + </relation> + <relation id="31640" version="81" timestamp="2012-05-19T09:17:44Z" uid="24119" user="Mauls" changeset="11640673"> + <member type="way" ref="24541150" role=""/> + <member type="way" ref="25896432" role="forward"/> + <member type="way" ref="25896435" role="forward"/> + <member type="way" ref="136990875" role="forward"/> + <member type="way" ref="25896366" role="forward"/> + <member type="way" ref="25896438" role="forward"/> + <member type="way" ref="136990877" role="forward"/> + <member type="way" ref="22329168" role="forward"/> + <member type="way" ref="136990870" role=""/> + <member type="way" ref="136990880" role=""/> + <member type="way" ref="136990882" role=""/> + <member type="way" ref="3220127" role=""/> + <member type="way" ref="136990873" role=""/> + <member type="way" ref="136990872" role=""/> + <member type="way" ref="121267851" role=""/> + <member type="way" ref="121267847" role=""/> + <member type="way" ref="4515378" role=""/> + <member type="way" ref="8145197" role=""/> + <member type="way" ref="19745207" role=""/> + <member type="way" ref="19745206" role=""/> + <member type="way" ref="4518667" role=""/> + <member type="way" ref="113946003" role=""/> + <member type="way" ref="136990884" role=""/> + <member type="way" ref="8126872" role=""/> + <member type="way" ref="1019866" role=""/> + <member type="way" ref="1935841" role=""/> + <member type="way" ref="1935842" role=""/> + <member type="way" ref="2837807" role=""/> + <member type="way" ref="3617718" role=""/> + <member type="way" ref="3617750" role=""/> + <member type="way" ref="157868716" role=""/> + <member type="way" ref="157868712" role=""/> + <member type="way" ref="4232426" role=""/> + <member type="way" ref="4232450" role=""/> + <member type="way" ref="4232523" role=""/> + <member type="way" ref="4234302" role=""/> + <member type="way" ref="4234304" role=""/> + <member type="way" ref="140969196" role=""/> + <member type="way" ref="4274177" role=""/> + <member type="way" ref="4288798" role=""/> + <member type="way" ref="4288804" role=""/> + <member type="way" ref="4288817" role=""/> + <member type="way" ref="115708422" role=""/> + <member type="way" ref="4290564" role=""/> + <member type="way" ref="4290566" role=""/> + <member type="way" ref="146405866" role=""/> + <member type="way" ref="4290567" role=""/> + <member type="way" ref="4290569" role=""/> + <member type="way" ref="104494931" role=""/> + <member type="way" ref="37726893" role=""/> + <member type="way" ref="4300013" role=""/> + <member type="way" ref="4327072" role=""/> + <member type="way" ref="4385337" role=""/> + <member type="way" ref="4515181" role=""/> + <member type="way" ref="4719529" role=""/> + <member type="way" ref="4719530" role=""/> + <member type="way" ref="5924137" role="forward"/> + <member type="way" ref="136990881" role=""/> + <member type="way" ref="135775543" role=""/> + <member type="way" ref="135775531" role=""/> + <member type="way" ref="6006524" role=""/> + <member type="way" ref="9227120" role=""/> + <member type="way" ref="9228211" role=""/> + <member type="way" ref="9228215" role=""/> + <member type="way" ref="9228221" role=""/> + <member type="way" ref="9228224" role=""/> + <member type="way" ref="9265930" role=""/> + <member type="way" ref="9362548" role=""/> + <member type="way" ref="10943268" role=""/> + <member type="way" ref="10943269" role=""/> + <member type="way" ref="10944242" role=""/> + <member type="way" ref="10989591" role=""/> + <member type="way" ref="10991833" role=""/> + <member type="way" ref="10996488" role=""/> + <member type="way" ref="11070521" role=""/> + <member type="way" ref="11071125" role=""/> + <member type="way" ref="71403686" role=""/> + <member type="way" ref="11071127" role=""/> + <member type="way" ref="11071128" role=""/> + <member type="way" ref="11071130" role=""/> + <member type="way" ref="11071828" role=""/> + <member type="way" ref="11071937" role=""/> + <member type="way" ref="11071938" role=""/> + <member type="way" ref="11071941" role=""/> + <member type="way" ref="11072077" role=""/> + <member type="way" ref="11072080" role=""/> + <member type="way" ref="15091285" role=""/> + <member type="way" ref="15091286" role=""/> + <member type="way" ref="15091321" role=""/> + <member type="way" ref="15091324" role=""/> + <member type="way" ref="15091328" role=""/> + <member type="way" ref="15091364" role=""/> + <member type="way" ref="15091378" role=""/> + <member type="way" ref="73271466" role=""/> + <member type="way" ref="73271488" role=""/> + <member type="way" ref="58718382" role=""/> + <member type="way" ref="15091546" role=""/> + <member type="way" ref="15091557" role=""/> + <member type="way" ref="15091559" role=""/> + <member type="way" ref="15091596" role=""/> + <member type="way" ref="15091603" role=""/> + <member type="way" ref="15091606" role=""/> + <member type="way" ref="48827325" role=""/> + <member type="way" ref="15091609" role=""/> + <member type="way" ref="164157963" role=""/> + <member type="way" ref="15091656" role=""/> + <member type="way" ref="43026009" role=""/> + <member type="way" ref="43026010" role=""/> + <member type="way" ref="15091678" role=""/> + <member type="way" ref="19826860" role=""/> + <member type="way" ref="19826861" role=""/> + <member type="way" ref="164157965" role=""/> + <member type="way" ref="19973630" role=""/> + <member type="way" ref="22277500" role=""/> + <member type="way" ref="22277501" role=""/> + <member type="way" ref="22277790" role=""/> + <member type="way" ref="22277845" role=""/> + <member type="way" ref="22277849" role=""/> + <member type="way" ref="22278089" role=""/> + <member type="way" ref="22278090" role=""/> + <member type="way" ref="22278109" role=""/> + <member type="way" ref="22278111" role=""/> + <member type="way" ref="22278116" role=""/> + <member type="way" ref="22278120" role=""/> + <member type="way" ref="22278128" role=""/> + <member type="way" ref="22278626" role=""/> + <member type="way" ref="22328765" role=""/> + <member type="way" ref="22328772" role=""/> + <member type="way" ref="22328778" role=""/> + <member type="way" ref="22328779" role=""/> + <member type="way" ref="22330396" role=""/> + <member type="way" ref="22679421" role=""/> + <member type="way" ref="23040738" role=""/> + <member type="way" ref="23040842" role=""/> + <member type="way" ref="23040846" role=""/> + <member type="way" ref="23040858" role=""/> + <member type="way" ref="23117246" role=""/> + <member type="way" ref="23294746" role=""/> + <member type="way" ref="23506818" role=""/> + <member type="way" ref="23506829" role=""/> + <member type="way" ref="23517268" role=""/> + <member type="way" ref="23517271" role=""/> + <member type="way" ref="157868707" role=""/> + <member type="way" ref="23598320" role=""/> + <member type="way" ref="23598322" role=""/> + <member type="way" ref="23598454" role=""/> + <member type="way" ref="23737552" role=""/> + <member type="way" ref="23737553" role=""/> + <member type="way" ref="23737667" role=""/> + <member type="way" ref="145738805" role=""/> + <member type="way" ref="24541552" role=""/> + <member type="way" ref="24541863" role=""/> + <member type="way" ref="24541865" role=""/> + <member type="way" ref="25681100" role=""/> + <member type="way" ref="25681103" role=""/> + <member type="way" ref="25681278" role=""/> + <member type="way" ref="25681279" role=""/> + <member type="way" ref="25681280" role=""/> + <member type="way" ref="25681403" role=""/> + <member type="way" ref="25681444" role=""/> + <member type="way" ref="25681445" role=""/> + <member type="way" ref="25690338" role=""/> + <member type="way" ref="25690339" role=""/> + <member type="way" ref="25690340" role=""/> + <member type="way" ref="25690749" role=""/> + <member type="way" ref="43023896" role=""/> + <member type="way" ref="43023897" role=""/> + <member type="way" ref="25690753" role=""/> + <member type="way" ref="25690754" role=""/> + <member type="way" ref="25690755" role=""/> + <member type="way" ref="25690809" role=""/> + <member type="way" ref="25690810" role=""/> + <member type="way" ref="38328051" role=""/> + <member type="way" ref="38328052" role=""/> + <member type="way" ref="25691524" role=""/> + <member type="way" ref="101712789" role=""/> + <member type="way" ref="101712803" role=""/> + <member type="way" ref="25692910" role=""/> + <member type="way" ref="25698340" role=""/> + <member type="way" ref="25698341" role=""/> + <member type="way" ref="25698342" role=""/> + <member type="way" ref="25698343" role=""/> + <member type="way" ref="25698344" role=""/> + <member type="way" ref="25698346" role=""/> + <member type="way" ref="25698348" role=""/> + <member type="way" ref="25698352" role=""/> + <member type="way" ref="25698353" role=""/> + <member type="way" ref="25698354" role=""/> + <member type="way" ref="25698355" role=""/> + <member type="way" ref="26263982" role=""/> + <member type="way" ref="26955717" role=""/> + <member type="way" ref="28366733" role=""/> + <member type="way" ref="33971858" role=""/> + <member type="way" ref="33971859" role=""/> + <member type="way" ref="145738820" role=""/> + <member type="way" ref="33995569" role=""/> + <member type="way" ref="23737666" role=""/> + <member type="way" ref="41093668" role=""/> + <member type="way" ref="44051303" role=""/> + <member type="way" ref="44131233" role=""/> + <member type="way" ref="44131234" role=""/> + <member type="way" ref="25690748" role=""/> + <member type="way" ref="25690752" role=""/> + <member type="way" ref="44033101" role=""/> + <member type="way" ref="44202299" role=""/> + <member type="way" ref="44317183" role=""/> + <member type="way" ref="44317184" role=""/> + <member type="way" ref="44834939" role=""/> + <member type="way" ref="45317529" role=""/> + <member type="way" ref="45675161" role=""/> + <member type="way" ref="4275500" role=""/> + <member type="way" ref="53574779" role=""/> + <member type="way" ref="55041560" role=""/> + <member type="way" ref="62941752" role=""/> + <member type="way" ref="70914841" role=""/> + <member type="way" ref="100443831" role=""/> + <member type="way" ref="103058540" role=""/> + <member type="way" ref="103058539" role=""/> + <member type="way" ref="25691035" role=""/> + <member type="way" ref="145809031" role=""/> + <member type="way" ref="140969180" role=""/> + <member type="way" ref="140969209" role=""/> + <member type="way" ref="10989259" role=""/> + <member type="way" ref="141570176" role=""/> + <member type="way" ref="157621618" role=""/> + <member type="way" ref="157868715" role=""/> + <member type="way" ref="157868706" role=""/> + <member type="way" ref="157868717" role=""/> + <member type="way" ref="157868710" role=""/> + <member type="way" ref="157868709" role=""/> + <member type="way" ref="157868714" role=""/> + <member type="way" ref="157868705" role=""/> + <member type="way" ref="157868718" role=""/> + <member type="way" ref="157868711" role=""/> + <tag k="name" v="NCN National Route 61"/> + <tag k="network" v="ncn"/> + <tag k="ref" v="61"/> + <tag k="route" v="bicycle"/> + <tag k="type" v="route"/> + </relation> + <relation id="267403" version="2" timestamp="2010-02-16T13:21:24Z" uid="91657" user="Pink Duck" changeset="3891683"> + <member type="node" ref="502550970" role=""/> + <member type="node" ref="502552090" role=""/> + <tag k="name" v="Oaktree Close"/> + <tag k="naptan:StopAreaCode" v="210G896"/> + <tag k="naptan:StopAreaType" v="GPBS"/> + <tag k="naptan:verified" v="no"/> + <tag k="site" v="stop_area"/> + <tag k="source" v="naptan_import"/> + <tag k="type" v="site"/> + </relation> + <relation id="267404" version="2" timestamp="2010-02-16T13:21:22Z" uid="91657" user="Pink Duck" changeset="3891683"> + <member type="node" ref="502550921" role=""/> + <member type="node" ref="502552074" role=""/> + <tag k="name" v="Burfield Close"/> + <tag k="naptan:StopAreaCode" v="210G897"/> + <tag k="naptan:StopAreaType" v="GPBS"/> + <tag k="naptan:verified" v="no"/> + <tag k="site" v="stop_area"/> + <tag k="source" v="naptan_import"/> + <tag k="type" v="site"/> + </relation> + <relation id="267400" version="2" timestamp="2010-02-16T13:21:23Z" uid="91657" user="Pink Duck" changeset="3891683"> + <member type="node" ref="502550963" role=""/> + <member type="node" ref="502552081" role=""/> + <tag k="name" v="Jasmine Gardens"/> + <tag k="naptan:StopAreaCode" v="210G895"/> + <tag k="naptan:StopAreaType" v="GPBS"/> + <tag k="naptan:verified" v="no"/> + <tag k="site" v="stop_area"/> + <tag k="source" v="naptan_import"/> + <tag k="type" v="site"/> + </relation> +</osm> +''' + +from xml.etree import ElementTree +from functools import reduce + +types = { + "way": "Way.class", + "node": "Node.class" +} + +tree = ElementTree.fromstring(relationxml) +for rel in tree.findall('relation'): + + memberTypes = [ types[m.get('type')] for m in rel.findall('member') ] + + memberIDs = [ m.get('ref') for m in rel.findall('member') ] + tags = [ [ '"'+m.get('k')+'"', '"'+m.get('v')+'"' ] for m in rel.findall('tag') ] + tags = reduce(list.__add__, tags) + + print(f'ExpectedRelation({rel.get("id")}, List.of({", ".join(memberTypes)}), List.of({"l, ".join(memberIDs)}l), List.of({", ".join(tags)})),') +\ No newline at end of file