From bb7706aeb06aa30f7e1339f3974a885f078dd870 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 19 Mar 2018 10:23:07 -0700 Subject: [PATCH 01/39] Fully integrated protobuf build flow with gradle. See ECLIPSE_README.md for instructions on how to get the project working with Eclipse. --- .gitignore | 1 + ECLIPSE_README.md | 39 ++++++++++++++++++ README.md | 3 ++ build.gradle | 41 +++++++++++++++++++ config/checkstyle/suppressions.xml | 3 +- dependencies.gradle | 2 + .../builder/proto/ProtoAtlasBuilder.java | 22 ++++++++++ src/main/proto/Test.proto | 12 ++++++ 8 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 ECLIPSE_README.md create mode 100644 src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java create mode 100644 src/main/proto/Test.proto diff --git a/.gitignore b/.gitignore index 2f5795517b..1d0d551115 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ **/out/ /bin/ **/ignored/ +gen/ diff --git a/ECLIPSE_README.md b/ECLIPSE_README.md new file mode 100644 index 0000000000..744ce3e8fc --- /dev/null +++ b/ECLIPSE_README.md @@ -0,0 +1,39 @@ +# Working with Atlas in Eclipse + +## First time setup +1. Clone the Atlas repository from your fork into the desired local location. + +2. Import the project into Eclipse using "File" -> "Import" -> "Gradle" -> "Existing Gradle Project" + +3. From the command line, run +``` +$ cd /path/to/local/Atlas/repo +$ ./gradlew clean build +``` + +4. Back in Eclipse, right click the Atlas project in Package Explorer and click "Refresh" + 1. NOTE: You should now see a "gen" folder appear in the package explorer + +5. Again, right click on the Atlas project in Package Explorer and select "Build Path" -> "Configure Build Path" + +6. Select "Add Folder" then expand "gen" -> "main" and select the box next to "java" + +7. Click "Apply and close" + +8. The red lines and X's should all be resolved now. You're good to go! + +## How to rebuild the protobuf templates +1. You're here because you wanted to change the .proto template files, and now you need to regenerate and rebuild the code in "gen". Easy! + +2. Make the necessary adjustment to the .proto files first (in src/main/proto), but do not touch any of the dependant source code. Otherwise Eclipse will start complaining. + +3. Once you are done adjusting the .proto files, open a command line and run +``` +$ cd /path/to/local/Atlas/repo +$ ./gradlew clean build +``` + +4. Back in Eclipse, right click on the Atlas project in Package Explorer and click "Refresh" + +5. Now you can make changes to your actual source, which depends on the code generated from the .proto files. The Eclipse play button should work once again. + diff --git a/README.md b/README.md index 68348f1088..f8c90bccbb 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,9 @@ final Atlas atlas2; new PackedAtlasCloner().cloneFrom(new MultiAtlas(atlas1, atlas2)).save(new File("/path/to/file.atlas")); ``` +## Using `Atlas` in Eclipse +See the instructions in [ECLIPSE_README.md](ECLIPSE_REAMDE.md) before attempting to integrate the `Atlas` project with Eclipse. Some extra setup is required to get the Eclipse playbutton to work. + # Implementation Details ## [`PackedAtlas`](src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlas.java) diff --git a/build.gradle b/build.gradle index 41090d10f0..c16b3aa7ac 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,7 @@ buildscript { dependencies { classpath 'com.diffplug.spotless:spotless-plugin-gradle:3.4.0' + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.1' } } @@ -10,6 +11,7 @@ buildscript { plugins { id "io.codearte.nexus-staging" version "0.8.0" + id "com.google.protobuf" version "0.8.1" } apply from: 'dependencies.gradle' @@ -19,6 +21,7 @@ apply plugin: 'maven' apply plugin: 'signing' apply plugin: "com.diffplug.gradle.spotless" apply plugin: 'idea' +apply plugin: 'com.google.protobuf' sourceCompatibility=1.8 targetCompatibility=1.8 @@ -71,6 +74,13 @@ sourceSets } resources.srcDir file('src/integrationTest/resources') } + main + { + java + { + srcDir file('gen/main/java') + } + } } test @@ -88,6 +98,36 @@ configurations integrationTestRuntime.extendsFrom testRuntime } +protobuf { + + generatedFilesBaseDir = 'gen/' + + protoc { + artifact = 'com.google.protobuf:protoc:2.6.1' + } + + generateProtoTasks { + all().each { task -> + if (task.name == "generateProto") { + task.outputs.upToDateWhen { + new File(generatedFilesBaseDir).exists() + } + } + } + + task deleteGeneratedProto { + delete generatedFilesBaseDir + } + + task recompileProto {} + recompileProto.dependsOn(deleteGeneratedProto) + recompileProto.dependsOn(generateProto) + + compileJava.dependsOn(generateProto) + clean.dependsOn(deleteGeneratedProto) + } +} + dependencies { compile packages.slf4j.api @@ -116,6 +156,7 @@ dependencies compile packages.jsonassert compile packages.jackson.core compile packages.jackson.databind + compile packages.protobuf_java shaded project.configurations.getByName('compile') shaded packages.slf4j.log4j12 diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index 7ae87bfa6a..54c5173446 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -9,5 +9,6 @@ - + + diff --git a/dependencies.gradle b/dependencies.gradle index 4bc4f26f67..3cf606c0ce 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -20,6 +20,7 @@ project.ext.versions = [ jsonassert:'1.3.0', jackson_core:'2.8.8', jackson_databind:'2.8.8.1', + protobuf_java:'2.6.1', ] project.ext.packages = [ @@ -56,5 +57,6 @@ project.ext.packages = [ core: "com.fasterxml.jackson.core:jackson-core:${versions.jackson_core}", databind: "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" ], + protobuf_java: "com.google.protobuf:protobuf-java:2.6.1" ] diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java new file mode 100644 index 0000000000..778e8c7e0f --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -0,0 +1,22 @@ +package org.openstreetmap.atlas.geography.atlas.builder.proto; + +import org.openstreetmap.atlas.proto.Test; + +/** + * Build an Atlas from a primitive proto serialized file, or write an Atlas to a proto serialized + * file. + * + * @author lcram + */ +public final class ProtoAtlasBuilder +{ + public static void main(final String[] args) + { + final Test test = Test.newBuilder().setField1("test").setField2("asd").build(); + System.out.println(test); + } + + private ProtoAtlasBuilder() + { + } +} diff --git a/src/main/proto/Test.proto b/src/main/proto/Test.proto new file mode 100644 index 0000000000..e48645ffcf --- /dev/null +++ b/src/main/proto/Test.proto @@ -0,0 +1,12 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "TestProto"; + +package org.openstreetmap.atlas.proto; + +message Test { + optional string field1 = 1; + optional string field2 = 2; +} + From 1890bc843c34e9367a1c79dc9e89994d07632d15 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 19 Mar 2018 14:32:40 -0700 Subject: [PATCH 02/39] Fixed spotless application to ignore generated source code. --- build.gradle | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index c16b3aa7ac..8d39593fac 100644 --- a/build.gradle +++ b/build.gradle @@ -40,11 +40,15 @@ idea { } spotless { - java { - importOrder(['static java', 'static javax', 'static org', 'static com', 'static scala', 'java', 'javax', 'org', 'com', 'scala']) - removeUnusedImports() - eclipse().configFile 'config/format/code_format.xml' - } + java { + target project.fileTree(project.rootDir) { + include '**/*.java' + exclude 'gen/**/*.java' + } + importOrder(['static java', 'static javax', 'static org', 'static com', 'static scala', 'java', 'javax', 'org', 'com', 'scala']) + removeUnusedImports() + eclipse().configFile 'config/format/code_format.xml' + } } // corresponds to POM description From a03cadef5f21a43548a620f47d18bfb61df2162b Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 19 Mar 2018 15:17:35 -0700 Subject: [PATCH 03/39] Initial, naive implementation of proto specs for Tags and Points. --- .../atlas/builder/proto/ProtoAtlasBuilder.java | 13 +------------ src/main/proto/Point.proto | 16 ++++++++++++++++ src/main/proto/Tag.proto | 10 ++++++++++ src/main/proto/Test.proto | 12 ------------ 4 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 src/main/proto/Point.proto create mode 100644 src/main/proto/Tag.proto delete mode 100644 src/main/proto/Test.proto diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 778e8c7e0f..5923e20183 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -1,22 +1,11 @@ package org.openstreetmap.atlas.geography.atlas.builder.proto; -import org.openstreetmap.atlas.proto.Test; - /** * Build an Atlas from a primitive proto serialized file, or write an Atlas to a proto serialized * file. * * @author lcram */ -public final class ProtoAtlasBuilder +public class ProtoAtlasBuilder { - public static void main(final String[] args) - { - final Test test = Test.newBuilder().setField1("test").setField2("asd").build(); - System.out.println(test); - } - - private ProtoAtlasBuilder() - { - } } diff --git a/src/main/proto/Point.proto b/src/main/proto/Point.proto new file mode 100644 index 0000000000..601fb7055a --- /dev/null +++ b/src/main/proto/Point.proto @@ -0,0 +1,16 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoPointWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Tag.proto"; + +message ProtoPoint { + optional int64 id = 1; + optional int32 latitude = 2; + optional int32 longitude = 3; + + repeated ProtoTag tags = 4; +} diff --git a/src/main/proto/Tag.proto b/src/main/proto/Tag.proto new file mode 100644 index 0000000000..19bea42ed7 --- /dev/null +++ b/src/main/proto/Tag.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoTagWrapper"; + +package org.openstreetmap.atlas.proto; + +message ProtoTag { + optional string tagText = 1; +} diff --git a/src/main/proto/Test.proto b/src/main/proto/Test.proto deleted file mode 100644 index e48645ffcf..0000000000 --- a/src/main/proto/Test.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto2"; - -option java_multiple_files = true; -option java_outer_classname = "TestProto"; - -package org.openstreetmap.atlas.proto; - -message Test { - optional string field1 = 1; - optional string field2 = 2; -} - From 4e05e34256062bf5081d573468498d48fcd61b2a Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Tue, 20 Mar 2018 13:12:14 -0700 Subject: [PATCH 04/39] Small updates to the README files. --- ECLIPSE_README.md | 4 ++-- README.md | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ECLIPSE_README.md b/ECLIPSE_README.md index 744ce3e8fc..119a0ab785 100644 --- a/ECLIPSE_README.md +++ b/ECLIPSE_README.md @@ -8,7 +8,7 @@ 3. From the command line, run ``` $ cd /path/to/local/Atlas/repo -$ ./gradlew clean build +$ ./gradlew clean build -x test -x integrationTest ``` 4. Back in Eclipse, right click the Atlas project in Package Explorer and click "Refresh" @@ -30,7 +30,7 @@ $ ./gradlew clean build 3. Once you are done adjusting the .proto files, open a command line and run ``` $ cd /path/to/local/Atlas/repo -$ ./gradlew clean build +$ ./gradlew clean build -x test -x integrationTest ``` 4. Back in Eclipse, right click on the Atlas project in Package Explorer and click "Refresh" diff --git a/README.md b/README.md index f8c90bccbb..68348f1088 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,6 @@ final Atlas atlas2; new PackedAtlasCloner().cloneFrom(new MultiAtlas(atlas1, atlas2)).save(new File("/path/to/file.atlas")); ``` -## Using `Atlas` in Eclipse -See the instructions in [ECLIPSE_README.md](ECLIPSE_REAMDE.md) before attempting to integrate the `Atlas` project with Eclipse. Some extra setup is required to get the Eclipse playbutton to work. - # Implementation Details ## [`PackedAtlas`](src/main/java/org/openstreetmap/atlas/geography/atlas/packed/PackedAtlas.java) From 18611a39030f5992fb3b2c131ef9d076baa52e65 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Tue, 20 Mar 2018 15:16:06 -0700 Subject: [PATCH 05/39] Started implementation of naive ProtoAtlasBuilder --- .../builder/proto/ProtoAtlasBuilder.java | 74 ++++++++++++++++++- src/main/proto/AtlasContainer.proto | 12 +++ .../builder/proto/ProtoAtlasBuilderTest.java | 37 ++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 src/main/proto/AtlasContainer.proto create mode 100644 src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 5923e20183..acae6f705e 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -1,11 +1,83 @@ package org.openstreetmap.atlas.geography.atlas.builder.proto; +import java.util.Map; + +import org.openstreetmap.atlas.exception.CoreException; +import org.openstreetmap.atlas.geography.atlas.Atlas; +import org.openstreetmap.atlas.geography.atlas.items.Point; +import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; +import org.openstreetmap.atlas.proto.ProtoAtlasContainer; +import org.openstreetmap.atlas.proto.ProtoPoint; +import org.openstreetmap.atlas.proto.ProtoTag; +import org.openstreetmap.atlas.streaming.resource.Resource; +import org.openstreetmap.atlas.streaming.resource.WritableResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** - * Build an Atlas from a primitive proto serialized file, or write an Atlas to a proto serialized + * Build an Atlas from a naive proto serialized file, or write an Atlas to a naive proto serialized * file. * * @author lcram */ public class ProtoAtlasBuilder { + private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilder.class); + + private static final String TAG_DELIMITER = "="; + + public PackedAtlas read(final Resource resource) + { + final long numberOfNodes = 0L; + final long numberOfEdges = 0L; + final long numberOfAreas = 0L; + final long numberOfLines = 0L; + final long numberOfPoints = 0L; + final long numberOfRelations = 0L; + + return null; + } + + public void write(final Atlas atlas, final WritableResource resource) + { + final ProtoAtlasContainer.Builder protoAtlasBuilder = ProtoAtlasContainer.newBuilder(); + for (final Point point : atlas.points()) + { + logger.debug("PROCESSING POINT:\n" + point.toString()); + final ProtoPoint.Builder protoPointBuilder = ProtoPoint.newBuilder(); + protoPointBuilder.setId(point.getIdentifier()); + + // TODO We have to cast to an int here, since asDm7() returns a long. + // To do this safely, we use Math.toIntExact, which throws an ArithmeticException + // when the long is out of range. Here I have caught it and rethrown a CoreException. + // Should I just leave it to throw the ArithmeticException? + try + { + protoPointBuilder + .setLatitude(Math.toIntExact(point.getLocation().getLatitude().asDm7())); + protoPointBuilder + .setLongitude(Math.toIntExact(point.getLocation().getLongitude().asDm7())); + } + catch (final ArithmeticException exception) + { + throw new CoreException( + "Point " + point.toString() + " had latlong out of int32 range."); + } + + final Map tags = point.getTags(); + logger.debug("tag set"); + for (final Map.Entry entry : tags.entrySet()) + { + final ProtoTag.Builder tagBuilder = ProtoTag.newBuilder(); + final String fullTag = entry.getKey() + TAG_DELIMITER + entry.getValue(); + logger.debug(fullTag); + tagBuilder.setTagText(fullTag); + protoPointBuilder.addTags(tagBuilder.build()); + } + + protoAtlasBuilder.addPoints(protoPointBuilder.build()); + + // TODO how do you write a resource to a file? + } + } } diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto new file mode 100644 index 0000000000..c712e9568e --- /dev/null +++ b/src/main/proto/AtlasContainer.proto @@ -0,0 +1,12 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoAtlasContainerWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Point.proto"; + +message ProtoAtlasContainer { + repeated ProtoPoint points = 1; +} \ No newline at end of file diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java new file mode 100644 index 0000000000..daf094ffa7 --- /dev/null +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -0,0 +1,37 @@ +package org.openstreetmap.atlas.geography.atlas.builder.proto; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; +import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.atlas.Atlas; +import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; +import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; +import org.openstreetmap.atlas.streaming.resource.WritableResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author lcram + */ +public class ProtoAtlasBuilderTest +{ + private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilderTest.class); + + @Test + public void testWriteFunctionality() + { + final PackedAtlasBuilder builder = new PackedAtlasBuilder(); + final Map tags = new HashMap<>(); + tags.put("building", "yes"); + tags.put("name", "eiffel_tower"); + builder.addPoint(0, Location.EIFFEL_TOWER, tags); + + final Atlas atlas = builder.get(); + final WritableResource resource = new ByteArrayResource(); + + final ProtoAtlasBuilder protoAtlasBuilder = new ProtoAtlasBuilder(); + protoAtlasBuilder.write(atlas, resource); + } +} From f3824667efc13723016585f18a853042827f3349 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Wed, 21 Mar 2018 16:19:28 -0700 Subject: [PATCH 06/39] Implemented Point and Line packing. --- .../builder/proto/ProtoAtlasBuilder.java | 230 +++++++++++++++--- src/main/proto/AtlasContainer.proto | 7 +- src/main/proto/Line.proto | 16 ++ src/main/proto/Location.proto | 13 + src/main/proto/Point.proto | 6 +- .../builder/proto/ProtoAtlasBuilderTest.java | 40 ++- 6 files changed, 263 insertions(+), 49 deletions(-) create mode 100644 src/main/proto/Line.proto create mode 100644 src/main/proto/Location.proto diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index acae6f705e..fa27e160d6 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -1,22 +1,40 @@ package org.openstreetmap.atlas.geography.atlas.builder.proto; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; +import org.openstreetmap.atlas.geography.Latitude; +import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.Longitude; +import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; +import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; +import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; +import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; +import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.proto.ProtoAtlasContainer; +import org.openstreetmap.atlas.proto.ProtoLine; +import org.openstreetmap.atlas.proto.ProtoLocation; import org.openstreetmap.atlas.proto.ProtoPoint; import org.openstreetmap.atlas.proto.ProtoTag; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; +import org.openstreetmap.atlas.utilities.collections.Maps; +import org.openstreetmap.atlas.utilities.collections.StringList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.protobuf.InvalidProtocolBufferException; + /** * Build an Atlas from a naive proto serialized file, or write an Atlas to a naive proto serialized - * file. + * file. At this stage, the builder can only handle Atlases that have tagged Points and Lines. * * @author lcram */ @@ -26,58 +44,194 @@ public class ProtoAtlasBuilder private static final String TAG_DELIMITER = "="; + /** + * Read a resource in naive ProtoAtlas format into a PackedAtlas. + * + * @param resource + * the resource in naive ProtoAtlas format + * @return the constructed PackedAtlas + */ public PackedAtlas read(final Resource resource) { - final long numberOfNodes = 0L; - final long numberOfEdges = 0L; - final long numberOfAreas = 0L; - final long numberOfLines = 0L; - final long numberOfPoints = 0L; - final long numberOfRelations = 0L; - - return null; + ProtoAtlasContainer protoAtlasContainer = null; + + // First, we need to construct the container object from the proto binary + try + { + protoAtlasContainer = ProtoAtlasContainer.parseFrom(resource.readBytesAndClose()); + } + // TODO do something smarter here? + catch (final InvalidProtocolBufferException exception) + { + throw new CoreException("Error parsing the protobuf"); + } + + // initialize atlas metadata + final long numberOfPoints = protoAtlasContainer.getNumberOfPoints(); + final long numberOfLines = protoAtlasContainer.getNumberOfLines(); + final long numberOfAreas = 0; + final long numberOfNodes = 0; + final long numberOfEdges = 0; + final long numberOfRelations = 0; + final AtlasSize atlasSize = new AtlasSize(numberOfEdges, numberOfNodes, numberOfAreas, + numberOfLines, numberOfPoints, numberOfRelations); + final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, true, "unknown", + "ProtoAtlas", "unknown", "unknown", Maps.hashMap()); + final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(atlasSize) + .withMetaData(atlasMetaData).withName(resource.getName()); + + // build the atlas features + parsePoints(builder, protoAtlasContainer.getPointsList()); + parseLines(builder, protoAtlasContainer.getLinesList()); + + return (PackedAtlas) builder.get(); } + /** + * Write an Atlas to a resource in the naive ProtoAtlas format. + * + * @param atlas + * the Atlas to be written + * @param resource + * the resource to write into + */ public void write(final Atlas atlas, final WritableResource resource) { final ProtoAtlasContainer.Builder protoAtlasBuilder = ProtoAtlasContainer.newBuilder(); - for (final Point point : atlas.points()) + + writePointsToBuilder(atlas, protoAtlasBuilder); + writeLinesToBuilder(atlas, protoAtlasBuilder); + + final ProtoAtlasContainer protoAtlas = protoAtlasBuilder.build(); + resource.writeAndClose(protoAtlas.toByteArray()); + } + + private List buildProtoTagListFromTagMap(final Map tags) + { + final List protoTags = new ArrayList<>(); + for (final Map.Entry entry : tags.entrySet()) { - logger.debug("PROCESSING POINT:\n" + point.toString()); - final ProtoPoint.Builder protoPointBuilder = ProtoPoint.newBuilder(); - protoPointBuilder.setId(point.getIdentifier()); + final ProtoTag.Builder tagBuilder = ProtoTag.newBuilder(); - // TODO We have to cast to an int here, since asDm7() returns a long. - // To do this safely, we use Math.toIntExact, which throws an ArithmeticException - // when the long is out of range. Here I have caught it and rethrown a CoreException. - // Should I just leave it to throw the ArithmeticException? - try - { - protoPointBuilder - .setLatitude(Math.toIntExact(point.getLocation().getLatitude().asDm7())); - protoPointBuilder - .setLongitude(Math.toIntExact(point.getLocation().getLongitude().asDm7())); - } - catch (final ArithmeticException exception) + // TODO what happens if entry.getValue() returns null? + // this could happen if someone inserted a null entry at that key + // maybe use entry.getValue() != null ? entry.getValue() : "" + final String fullTag = entry.getKey() + TAG_DELIMITER + entry.getValue(); + tagBuilder.setTagText(fullTag); + protoTags.add(tagBuilder.build()); + } + return protoTags; + } + + private ProtoLocation convertLocationToProtoLocation(final Location location) + { + final ProtoLocation.Builder protoLocationBuilder = ProtoLocation.newBuilder(); + protoLocationBuilder.setLatitude(Math.toIntExact(location.getLatitude().asDm7())); + protoLocationBuilder.setLongitude(Math.toIntExact(location.getLongitude().asDm7())); + return protoLocationBuilder.build(); + } + + private Location convertProtoLocationToLocation(final ProtoLocation protoLocation) + { + final Longitude longitude = Longitude.dm7(protoLocation.getLongitude()); + final Latitude latitude = Latitude.dm7(protoLocation.getLatitude()); + return new Location(latitude, longitude); + } + + private void parseLines(final PackedAtlasBuilder builder, final List lines) + { + lines.forEach(line -> + { + final long identifier = line.getId(); + final List shapePoints = line.getShapePointsList().stream() + .map(this::convertProtoLocationToLocation).collect(Collectors.toList()); + final PolyLine geometry = new PolyLine(shapePoints); + final Map tags = parseTags(line.getTagsList()); + builder.addLine(identifier, geometry, tags); + }); + } + + private void parsePoints(final PackedAtlasBuilder builder, final List points) + { + points.forEach(point -> + { + final long identifier = point.getId(); + final Longitude longitude = Longitude.dm7(point.getLocation().getLongitude()); + final Latitude latitude = Latitude.dm7(point.getLocation().getLatitude()); + final Location geometry = new Location(latitude, longitude); + final Map tags = parseTags(point.getTagsList()); + builder.addPoint(identifier, geometry, tags); + }); + } + + private Map parseTags(final List tagsList) + { + try + { + final Map result = Maps.hashMap(); + for (final ProtoTag tag : tagsList) { - throw new CoreException( - "Point " + point.toString() + " had latlong out of int32 range."); + final StringList tagSplit = StringList.split(tag.getTagText(), TAG_DELIMITER); + if (tagSplit.size() == 2) + { + result.put(tagSplit.get(0), tagSplit.get(1)); + } + if (tagSplit.size() == 1) + { + result.put(tagSplit.get(0), ""); + } } + return result; + } + // TODO do something smarter here? + catch (final Throwable error) + { + throw new CoreException("Unable to parse tags."); + } + } + + private void writeLinesToBuilder(final Atlas atlas, + final ProtoAtlasContainer.Builder protoAtlasBuilder) + { + long numberOfLines = 0; + + for (final Line line : atlas.lines()) + { + final ProtoLine.Builder protoLineBuilder = ProtoLine.newBuilder(); + protoLineBuilder.setId(line.getIdentifier()); + + final List protoLocations = StreamSupport + .stream(line.getRawGeometry().spliterator(), false) + .map(this::convertLocationToProtoLocation).collect(Collectors.toList()); + protoLineBuilder.addAllShapePoints(protoLocations); + + final Map tags = line.getTags(); + protoLineBuilder.addAllTags(buildProtoTagListFromTagMap(tags)); + + numberOfLines++; + protoAtlasBuilder.addLines(protoLineBuilder.build()); + } + protoAtlasBuilder.setNumberOfLines(numberOfLines); + } + + private void writePointsToBuilder(final Atlas atlas, + final ProtoAtlasContainer.Builder protoAtlasBuilder) + { + long numberOfPoints = 0; + + for (final Point point : atlas.points()) + { + final ProtoPoint.Builder protoPointBuilder = ProtoPoint.newBuilder(); + + protoPointBuilder.setId(point.getIdentifier()); + protoPointBuilder.setLocation(convertLocationToProtoLocation(point.getLocation())); final Map tags = point.getTags(); - logger.debug("tag set"); - for (final Map.Entry entry : tags.entrySet()) - { - final ProtoTag.Builder tagBuilder = ProtoTag.newBuilder(); - final String fullTag = entry.getKey() + TAG_DELIMITER + entry.getValue(); - logger.debug(fullTag); - tagBuilder.setTagText(fullTag); - protoPointBuilder.addTags(tagBuilder.build()); - } + protoPointBuilder.addAllTags(buildProtoTagListFromTagMap(tags)); + numberOfPoints++; protoAtlasBuilder.addPoints(protoPointBuilder.build()); - - // TODO how do you write a resource to a file? } + protoAtlasBuilder.setNumberOfPoints(numberOfPoints); } } diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto index c712e9568e..3e39de25f3 100644 --- a/src/main/proto/AtlasContainer.proto +++ b/src/main/proto/AtlasContainer.proto @@ -5,8 +5,13 @@ option java_outer_classname = "ProtoAtlasContainerWrapper"; package org.openstreetmap.atlas.proto; +import "Line.proto"; import "Point.proto"; message ProtoAtlasContainer { - repeated ProtoPoint points = 1; + optional int64 numberOfPoints = 1; + repeated ProtoPoint points = 2; + + optional int64 numberOfLines = 3; + repeated ProtoLine lines = 4; } \ No newline at end of file diff --git a/src/main/proto/Line.proto b/src/main/proto/Line.proto new file mode 100644 index 0000000000..8fc8994b91 --- /dev/null +++ b/src/main/proto/Line.proto @@ -0,0 +1,16 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoLineWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Location.proto"; +import "Tag.proto"; + +message ProtoLine { + optional int64 id = 1; + repeated ProtoLocation shapePoints = 2; + + repeated ProtoTag tags = 3; +} diff --git a/src/main/proto/Location.proto b/src/main/proto/Location.proto new file mode 100644 index 0000000000..63692f364a --- /dev/null +++ b/src/main/proto/Location.proto @@ -0,0 +1,13 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoLocationWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Tag.proto"; + +message ProtoLocation { + optional int32 latitude = 1; + optional int32 longitude = 2; +} diff --git a/src/main/proto/Point.proto b/src/main/proto/Point.proto index 601fb7055a..d74d83de05 100644 --- a/src/main/proto/Point.proto +++ b/src/main/proto/Point.proto @@ -5,12 +5,12 @@ option java_outer_classname = "ProtoPointWrapper"; package org.openstreetmap.atlas.proto; +import "Location.proto"; import "Tag.proto"; message ProtoPoint { optional int64 id = 1; - optional int32 latitude = 2; - optional int32 longitude = 3; + optional ProtoLocation location = 2; - repeated ProtoTag tags = 4; + repeated ProtoTag tags = 3; } diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index daf094ffa7..8f868ccc85 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -1,10 +1,14 @@ package org.openstreetmap.atlas.geography.atlas.builder.proto; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; @@ -20,18 +24,40 @@ public class ProtoAtlasBuilderTest private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilderTest.class); @Test - public void testWriteFunctionality() + public void testReadWriteConsistency() { - final PackedAtlasBuilder builder = new PackedAtlasBuilder(); + final WritableResource resource = new ByteArrayResource(); + final ProtoAtlasBuilder protoAtlasBuilder = new ProtoAtlasBuilder(); + final PackedAtlasBuilder packedAtlasBuilder = setUpTestAtlasBuilder(); + + // make sure the atlases are the same + final Atlas outAtlas = packedAtlasBuilder.get(); + protoAtlasBuilder.write(outAtlas, resource); + final Atlas inAtlas = protoAtlasBuilder.read(resource); + Assert.assertEquals(outAtlas, inAtlas); + } + + private PackedAtlasBuilder setUpTestAtlasBuilder() + { + final PackedAtlasBuilder packedAtlasBuilder = new PackedAtlasBuilder(); final Map tags = new HashMap<>(); + tags.put("building", "yes"); tags.put("name", "eiffel_tower"); - builder.addPoint(0, Location.EIFFEL_TOWER, tags); + packedAtlasBuilder.addPoint(0, Location.EIFFEL_TOWER, tags); - final Atlas atlas = builder.get(); - final WritableResource resource = new ByteArrayResource(); + tags.clear(); + tags.put("building", "yes"); + tags.put("name", "colosseum"); + packedAtlasBuilder.addPoint(1, Location.COLOSSEUM, tags); - final ProtoAtlasBuilder protoAtlasBuilder = new ProtoAtlasBuilder(); - protoAtlasBuilder.write(atlas, resource); + tags.clear(); + tags.put("path", "yes"); + final List shapePoints = new ArrayList<>(); + shapePoints.add(Location.EIFFEL_TOWER); + shapePoints.add(Location.COLOSSEUM); + packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); + + return packedAtlasBuilder; } } From fd9af273c673b96dd99b674dee8d6abb11926118 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 09:45:24 -0700 Subject: [PATCH 07/39] Stylistic and structural changes per PR notes. --- .../builder/proto/ProtoAtlasBuilder.java | 110 +++++------------- .../conversion/ProtoLocationConverter.java | 31 +++++ .../conversion/ProtoTagListConverter.java | 89 ++++++++++++++ 3 files changed, 148 insertions(+), 82 deletions(-) create mode 100644 src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java create mode 100644 src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index fa27e160d6..b8c753cf2e 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -1,10 +1,8 @@ package org.openstreetmap.atlas.geography.atlas.builder.proto; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.geography.Latitude; @@ -22,11 +20,11 @@ import org.openstreetmap.atlas.proto.ProtoLine; import org.openstreetmap.atlas.proto.ProtoLocation; import org.openstreetmap.atlas.proto.ProtoPoint; -import org.openstreetmap.atlas.proto.ProtoTag; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Maps; -import org.openstreetmap.atlas.utilities.collections.StringList; +import org.openstreetmap.atlas.utilities.conversion.ProtoLocationConverter; +import org.openstreetmap.atlas.utilities.conversion.ProtoTagListConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,8 +40,6 @@ public class ProtoAtlasBuilder { private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilder.class); - private static final String TAG_DELIMITER = "="; - /** * Read a resource in naive ProtoAtlas format into a PackedAtlas. * @@ -60,10 +56,9 @@ public PackedAtlas read(final Resource resource) { protoAtlasContainer = ProtoAtlasContainer.parseFrom(resource.readBytesAndClose()); } - // TODO do something smarter here? catch (final InvalidProtocolBufferException exception) { - throw new CoreException("Error parsing the protobuf"); + throw new CoreException("Error deserializing the ProtoAtlasContainer", exception); } // initialize atlas metadata @@ -75,6 +70,8 @@ public PackedAtlas read(final Resource resource) final long numberOfRelations = 0; final AtlasSize atlasSize = new AtlasSize(numberOfEdges, numberOfNodes, numberOfAreas, numberOfLines, numberOfPoints, numberOfRelations); + // TODO it would be nice to programmatically decide which version of protobuf is being used + // and store that with the atlas data final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, true, "unknown", "ProtoAtlas", "unknown", "unknown", Maps.hashMap()); final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(atlasSize) @@ -106,107 +103,53 @@ public void write(final Atlas atlas, final WritableResource resource) resource.writeAndClose(protoAtlas.toByteArray()); } - private List buildProtoTagListFromTagMap(final Map tags) - { - final List protoTags = new ArrayList<>(); - for (final Map.Entry entry : tags.entrySet()) - { - final ProtoTag.Builder tagBuilder = ProtoTag.newBuilder(); - - // TODO what happens if entry.getValue() returns null? - // this could happen if someone inserted a null entry at that key - // maybe use entry.getValue() != null ? entry.getValue() : "" - final String fullTag = entry.getKey() + TAG_DELIMITER + entry.getValue(); - tagBuilder.setTagText(fullTag); - protoTags.add(tagBuilder.build()); - } - return protoTags; - } - - private ProtoLocation convertLocationToProtoLocation(final Location location) - { - final ProtoLocation.Builder protoLocationBuilder = ProtoLocation.newBuilder(); - protoLocationBuilder.setLatitude(Math.toIntExact(location.getLatitude().asDm7())); - protoLocationBuilder.setLongitude(Math.toIntExact(location.getLongitude().asDm7())); - return protoLocationBuilder.build(); - } - - private Location convertProtoLocationToLocation(final ProtoLocation protoLocation) - { - final Longitude longitude = Longitude.dm7(protoLocation.getLongitude()); - final Latitude latitude = Latitude.dm7(protoLocation.getLatitude()); - return new Location(latitude, longitude); - } - private void parseLines(final PackedAtlasBuilder builder, final List lines) { - lines.forEach(line -> + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + lines.forEach(protoLine -> { - final long identifier = line.getId(); - final List shapePoints = line.getShapePointsList().stream() - .map(this::convertProtoLocationToLocation).collect(Collectors.toList()); + final long identifier = protoLine.getId(); + final List shapePoints = protoLine.getShapePointsList().stream() + .map(protoLocationConverter::convert).collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); - final Map tags = parseTags(line.getTagsList()); + final Map tags = protoTagListConverter.convert(protoLine.getTagsList()); builder.addLine(identifier, geometry, tags); }); } private void parsePoints(final PackedAtlasBuilder builder, final List points) { - points.forEach(point -> + final ProtoTagListConverter converter = new ProtoTagListConverter(); + points.forEach(protoPoint -> { - final long identifier = point.getId(); - final Longitude longitude = Longitude.dm7(point.getLocation().getLongitude()); - final Latitude latitude = Latitude.dm7(point.getLocation().getLatitude()); + final long identifier = protoPoint.getId(); + final Longitude longitude = Longitude.dm7(protoPoint.getLocation().getLongitude()); + final Latitude latitude = Latitude.dm7(protoPoint.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); - final Map tags = parseTags(point.getTagsList()); + final Map tags = converter.convert(protoPoint.getTagsList()); builder.addPoint(identifier, geometry, tags); }); } - private Map parseTags(final List tagsList) - { - try - { - final Map result = Maps.hashMap(); - for (final ProtoTag tag : tagsList) - { - final StringList tagSplit = StringList.split(tag.getTagText(), TAG_DELIMITER); - if (tagSplit.size() == 2) - { - result.put(tagSplit.get(0), tagSplit.get(1)); - } - if (tagSplit.size() == 1) - { - result.put(tagSplit.get(0), ""); - } - } - return result; - } - // TODO do something smarter here? - catch (final Throwable error) - { - throw new CoreException("Unable to parse tags."); - } - } - private void writeLinesToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { long numberOfLines = 0; + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Line line : atlas.lines()) { final ProtoLine.Builder protoLineBuilder = ProtoLine.newBuilder(); protoLineBuilder.setId(line.getIdentifier()); - final List protoLocations = StreamSupport - .stream(line.getRawGeometry().spliterator(), false) - .map(this::convertLocationToProtoLocation).collect(Collectors.toList()); + final List protoLocations = line.asPolyLine().stream() + .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); protoLineBuilder.addAllShapePoints(protoLocations); final Map tags = line.getTags(); - protoLineBuilder.addAllTags(buildProtoTagListFromTagMap(tags)); + protoLineBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); numberOfLines++; protoAtlasBuilder.addLines(protoLineBuilder.build()); @@ -218,16 +161,19 @@ private void writePointsToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { long numberOfPoints = 0; + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Point point : atlas.points()) { final ProtoPoint.Builder protoPointBuilder = ProtoPoint.newBuilder(); protoPointBuilder.setId(point.getIdentifier()); - protoPointBuilder.setLocation(convertLocationToProtoLocation(point.getLocation())); + protoPointBuilder + .setLocation(protoLocationConverter.backwardConvert(point.getLocation())); final Map tags = point.getTags(); - protoPointBuilder.addAllTags(buildProtoTagListFromTagMap(tags)); + protoPointBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); numberOfPoints++; protoAtlasBuilder.addPoints(protoPointBuilder.build()); diff --git a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java b/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java new file mode 100644 index 0000000000..f7c548d658 --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java @@ -0,0 +1,31 @@ +package org.openstreetmap.atlas.utilities.conversion; + +import org.openstreetmap.atlas.geography.Latitude; +import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.Longitude; +import org.openstreetmap.atlas.proto.ProtoLocation; + +/** + * Converts back and forth between ProtoLocation and Location + * + * @author lcram + */ +public class ProtoLocationConverter implements TwoWayConverter +{ + @Override + public ProtoLocation backwardConvert(final Location location) + { + final ProtoLocation.Builder protoLocationBuilder = ProtoLocation.newBuilder(); + protoLocationBuilder.setLatitude(Math.toIntExact(location.getLatitude().asDm7())); + protoLocationBuilder.setLongitude(Math.toIntExact(location.getLongitude().asDm7())); + return protoLocationBuilder.build(); + } + + @Override + public Location convert(final ProtoLocation protoLocation) + { + final Longitude longitude = Longitude.dm7(protoLocation.getLongitude()); + final Latitude latitude = Latitude.dm7(protoLocation.getLatitude()); + return new Location(latitude, longitude); + } +} diff --git a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java b/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java new file mode 100644 index 0000000000..4ddbe7bdbe --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java @@ -0,0 +1,89 @@ +package org.openstreetmap.atlas.utilities.conversion; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.openstreetmap.atlas.exception.CoreException; +import org.openstreetmap.atlas.proto.ProtoTag; +import org.openstreetmap.atlas.utilities.collections.Maps; +import org.openstreetmap.atlas.utilities.collections.StringList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Converts back and forth between a List of ProtoTags and an OSM tag Map. + * + * @author lcram + */ +public class ProtoTagListConverter implements TwoWayConverter, Map> +{ + private static final Logger logger = LoggerFactory.getLogger(ProtoTagListConverter.class); + + private static final String TAG_DELIMITER = "="; + + @Override + public List backwardConvert(final Map osmTagMap) + { + final List protoTags = new ArrayList<>(); + for (final Map.Entry entry : osmTagMap.entrySet()) + { + final ProtoTag.Builder tagBuilder = ProtoTag.newBuilder(); + final String keyText; + final String valueText; + + if (entry.getKey() == null) + { + logger.warn("buildProtoTagListFromTagMap found null key"); + keyText = "null"; + } + else + { + keyText = entry.getKey(); + } + + if (entry.getValue() == null) + { + logger.warn("buildProtoTagListFromTagMap found null value"); + valueText = "null"; + } + else + { + valueText = entry.getValue(); + } + + final String fullTag = keyText + TAG_DELIMITER + valueText; + tagBuilder.setTagText(fullTag); + protoTags.add(tagBuilder.build()); + + } + return protoTags; + } + + @Override + public Map convert(final List protoTagList) + { + try + { + final Map result = Maps.hashMap(); + for (final ProtoTag tag : protoTagList) + { + final StringList tagSplit = StringList.split(tag.getTagText(), TAG_DELIMITER); + if (tagSplit.size() == 2) + { + result.put(tagSplit.get(0), tagSplit.get(1)); + } + if (tagSplit.size() == 1) + { + result.put(tagSplit.get(0), ""); + } + } + return result; + } + catch (final Throwable error) + { + throw new CoreException("Unable to parse proto tags", error); + } + } + +} From fbdaaccd8ad7a9cb1eba2559b31871ba811e5c22 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 10:16:32 -0700 Subject: [PATCH 08/39] Trying to get Areas working with the serializer. --- .../builder/proto/ProtoAtlasBuilder.java | 44 +++++++++++++++++++ src/main/proto/Area.proto | 16 +++++++ src/main/proto/AtlasContainer.proto | 6 ++- .../builder/proto/ProtoAtlasBuilderTest.java | 12 ++++- 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 src/main/proto/Area.proto diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index b8c753cf2e..2f670e2abd 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -12,11 +12,14 @@ import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; +import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; +import org.openstreetmap.atlas.proto.ProtoArea; import org.openstreetmap.atlas.proto.ProtoAtlasContainer; +import org.openstreetmap.atlas.proto.ProtoAtlasContainer.Builder; import org.openstreetmap.atlas.proto.ProtoLine; import org.openstreetmap.atlas.proto.ProtoLocation; import org.openstreetmap.atlas.proto.ProtoPoint; @@ -80,6 +83,7 @@ public PackedAtlas read(final Resource resource) // build the atlas features parsePoints(builder, protoAtlasContainer.getPointsList()); parseLines(builder, protoAtlasContainer.getLinesList()); + parseAreas(builder, protoAtlasContainer.getAreasList()); return (PackedAtlas) builder.get(); } @@ -98,11 +102,27 @@ public void write(final Atlas atlas, final WritableResource resource) writePointsToBuilder(atlas, protoAtlasBuilder); writeLinesToBuilder(atlas, protoAtlasBuilder); + writeAreasToBuilder(atlas, protoAtlasBuilder); final ProtoAtlasContainer protoAtlas = protoAtlasBuilder.build(); resource.writeAndClose(protoAtlas.toByteArray()); } + private void parseAreas(final PackedAtlasBuilder builder, final List areas) + { + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + areas.forEach(protoArea -> + { + final long identifier = protoArea.getId(); + final List shapePoints = protoArea.getShapePointsList().stream() + .map(protoLocationConverter::convert).collect(Collectors.toList()); + final PolyLine geometry = new PolyLine(shapePoints); + final Map tags = protoTagListConverter.convert(protoArea.getTagsList()); + builder.addLine(identifier, geometry, tags); + }); + } + private void parseLines(final PackedAtlasBuilder builder, final List lines) { final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); @@ -132,6 +152,30 @@ private void parsePoints(final PackedAtlasBuilder builder, final List protoLocations = area.asPolygon().stream() + .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); + protoLineBuilder.addAllShapePoints(protoLocations); + + final Map tags = area.getTags(); + protoLineBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + + numberOfLines++; + protoAtlasBuilder.addLines(protoLineBuilder.build()); + } + protoAtlasBuilder.setNumberOfLines(numberOfLines); + } + private void writeLinesToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { diff --git a/src/main/proto/Area.proto b/src/main/proto/Area.proto new file mode 100644 index 0000000000..2be0975657 --- /dev/null +++ b/src/main/proto/Area.proto @@ -0,0 +1,16 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoAreaWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Location.proto"; +import "Tag.proto"; + +message ProtoArea { + optional int64 id = 1; + repeated ProtoLocation shapePoints = 2; + + repeated ProtoTag tags = 3; +} diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto index 3e39de25f3..68ad1f3430 100644 --- a/src/main/proto/AtlasContainer.proto +++ b/src/main/proto/AtlasContainer.proto @@ -5,8 +5,9 @@ option java_outer_classname = "ProtoAtlasContainerWrapper"; package org.openstreetmap.atlas.proto; -import "Line.proto"; import "Point.proto"; +import "Line.proto"; +import "Area.proto"; message ProtoAtlasContainer { optional int64 numberOfPoints = 1; @@ -14,4 +15,7 @@ message ProtoAtlasContainer { optional int64 numberOfLines = 3; repeated ProtoLine lines = 4; + + optional int64 numberOfAreas = 5; + repeated ProtoArea areas = 6; } \ No newline at end of file diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index 8f868ccc85..85ca05c3d4 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -41,6 +41,7 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() { final PackedAtlasBuilder packedAtlasBuilder = new PackedAtlasBuilder(); final Map tags = new HashMap<>(); + final List shapePoints = new ArrayList<>(); tags.put("building", "yes"); tags.put("name", "eiffel_tower"); @@ -53,11 +54,20 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() tags.clear(); tags.put("path", "yes"); - final List shapePoints = new ArrayList<>(); shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); + tags.clear(); + tags.put("triangle", "yes"); + tags.put("size", "stupidbig"); + shapePoints.clear(); + shapePoints.add(Location.EIFFEL_TOWER); + shapePoints.add(Location.COLOSSEUM); + shapePoints.add(Location.STEVENS_CREEK); + // TODO this is failing, why + // packedAtlasBuilder.addArea(3, new Polygon(shapePoints), tags); + return packedAtlasBuilder; } } From c9720feefb80deb0d04d60e8135c355e045ca7a9 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 11:16:03 -0700 Subject: [PATCH 09/39] Fixed tab indentation to use spaces in suppressions.xml --- config/checkstyle/suppressions.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index 54c5173446..5108491075 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -4,11 +4,11 @@ "http://www.puppycrawl.com/dtds/suppressions_1_1.dtd"> - - - - - + + + + + From bc1549683355cb7ff39f3055b4f604dbf99424cf Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 11:37:10 -0700 Subject: [PATCH 10/39] Trying to fix the Area serialization. Currently failing due to Polygon issues. --- .../atlas/builder/proto/ProtoAtlasBuilder.java | 7 +++++-- .../atlas/builder/proto/ProtoAtlasBuilderTest.java | 12 ++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 2f670e2abd..1f9c068d34 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -9,6 +9,7 @@ import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.geography.PolyLine; +import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; @@ -117,9 +118,11 @@ private void parseAreas(final PackedAtlasBuilder builder, final List final long identifier = protoArea.getId(); final List shapePoints = protoArea.getShapePointsList().stream() .map(protoLocationConverter::convert).collect(Collectors.toList()); - final PolyLine geometry = new PolyLine(shapePoints); + final Polygon geometry = new Polygon(shapePoints); final Map tags = protoTagListConverter.convert(protoArea.getTagsList()); - builder.addLine(identifier, geometry, tags); + // TODO see Mike's PR note about duplicate points, since start and end are the same for + // polygons, make sure this is not happening + builder.addArea(identifier, geometry, tags); }); } diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index 85ca05c3d4..accef22121 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -8,7 +8,7 @@ import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Location; -import org.openstreetmap.atlas.geography.PolyLine; +import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; @@ -45,18 +45,18 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() tags.put("building", "yes"); tags.put("name", "eiffel_tower"); - packedAtlasBuilder.addPoint(0, Location.EIFFEL_TOWER, tags); + // packedAtlasBuilder.addPoint(0, Location.EIFFEL_TOWER, tags); tags.clear(); tags.put("building", "yes"); tags.put("name", "colosseum"); - packedAtlasBuilder.addPoint(1, Location.COLOSSEUM, tags); + // packedAtlasBuilder.addPoint(1, Location.COLOSSEUM, tags); tags.clear(); tags.put("path", "yes"); shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); - packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); + // packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); tags.clear(); tags.put("triangle", "yes"); @@ -65,8 +65,8 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); shapePoints.add(Location.STEVENS_CREEK); - // TODO this is failing, why - // packedAtlasBuilder.addArea(3, new Polygon(shapePoints), tags); + // TODO this is failing due to point duplication issue Mike mentioned + packedAtlasBuilder.addArea(3, new Polygon(shapePoints), tags); return packedAtlasBuilder; } From a804b7225fdf67069c38d1d13c7fa6d38236c210 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 12:34:53 -0700 Subject: [PATCH 11/39] Fixed failing tests, which were due to bad copy paste. --- .../atlas/builder/proto/ProtoAtlasBuilder.java | 17 +++++++++-------- .../builder/proto/ProtoAtlasBuilderTest.java | 13 +++++++++---- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 1f9c068d34..09df855d36 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -74,8 +74,7 @@ public PackedAtlas read(final Resource resource) final long numberOfRelations = 0; final AtlasSize atlasSize = new AtlasSize(numberOfEdges, numberOfNodes, numberOfAreas, numberOfLines, numberOfPoints, numberOfRelations); - // TODO it would be nice to programmatically decide which version of protobuf is being used - // and store that with the atlas data + // TODO it would be nice to programmatically determine which protobuf version is in use final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, true, "unknown", "ProtoAtlas", "unknown", "unknown", Maps.hashMap()); final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(atlasSize) @@ -121,7 +120,9 @@ private void parseAreas(final PackedAtlasBuilder builder, final List final Polygon geometry = new Polygon(shapePoints); final Map tags = protoTagListConverter.convert(protoArea.getTagsList()); // TODO see Mike's PR note about duplicate points, since start and end are the same for - // polygons, make sure this is not happening + // polygons. Make sure this is not happening. I think it's fine since I am building the + // Polygon using the Polygon constructor from a list of points that contains no + // duplicates. builder.addArea(identifier, geometry, tags); }); } @@ -163,18 +164,18 @@ private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuil for (final Area area : atlas.areas()) { - final ProtoLine.Builder protoLineBuilder = ProtoLine.newBuilder(); - protoLineBuilder.setId(area.getIdentifier()); + final ProtoArea.Builder protoAreaBuilder = ProtoArea.newBuilder(); + protoAreaBuilder.setId(area.getIdentifier()); final List protoLocations = area.asPolygon().stream() .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); - protoLineBuilder.addAllShapePoints(protoLocations); + protoAreaBuilder.addAllShapePoints(protoLocations); final Map tags = area.getTags(); - protoLineBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoAreaBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); numberOfLines++; - protoAtlasBuilder.addLines(protoLineBuilder.build()); + protoAtlasBuilder.addAreas(protoAreaBuilder.build()); } protoAtlasBuilder.setNumberOfLines(numberOfLines); } diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index accef22121..8d998cea97 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -8,6 +8,7 @@ import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; @@ -34,6 +35,10 @@ public void testReadWriteConsistency() final Atlas outAtlas = packedAtlasBuilder.get(); protoAtlasBuilder.write(outAtlas, resource); final Atlas inAtlas = protoAtlasBuilder.read(resource); + + logger.debug("ATLAS THAT WAS WRITTEN OUT:\n{}\n", outAtlas); + logger.debug("ATLAS THAT WAS READ BACK:\n{}\n", inAtlas); + Assert.assertEquals(outAtlas, inAtlas); } @@ -45,18 +50,18 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() tags.put("building", "yes"); tags.put("name", "eiffel_tower"); - // packedAtlasBuilder.addPoint(0, Location.EIFFEL_TOWER, tags); + packedAtlasBuilder.addPoint(0, Location.EIFFEL_TOWER, tags); tags.clear(); tags.put("building", "yes"); tags.put("name", "colosseum"); - // packedAtlasBuilder.addPoint(1, Location.COLOSSEUM, tags); + packedAtlasBuilder.addPoint(1, Location.COLOSSEUM, tags); tags.clear(); tags.put("path", "yes"); shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); - // packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); + packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); tags.clear(); tags.put("triangle", "yes"); @@ -65,7 +70,7 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); shapePoints.add(Location.STEVENS_CREEK); - // TODO this is failing due to point duplication issue Mike mentioned + packedAtlasBuilder.addArea(3, new Polygon(shapePoints), tags); return packedAtlasBuilder; From 40437f7ba6cd2ddb925a4561887f5c2544085946 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 13:29:12 -0700 Subject: [PATCH 12/39] Integrated a few PR style suggestions. --- .../geography/atlas/builder/proto/ProtoAtlasBuilder.java | 3 ++- .../atlas/utilities/conversion/ProtoTagListConverter.java | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 09df855d36..f06aa8bd6d 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -62,7 +62,8 @@ public PackedAtlas read(final Resource resource) } catch (final InvalidProtocolBufferException exception) { - throw new CoreException("Error deserializing the ProtoAtlasContainer", exception); + throw new CoreException("Error deserializing the ProtoAtlasContainer from {}", + resource.getName(), exception); } // initialize atlas metadata diff --git a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java b/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java index 4ddbe7bdbe..65cf5fbba1 100644 --- a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java +++ b/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java @@ -34,8 +34,8 @@ public List backwardConvert(final Map osmTagMap) if (entry.getKey() == null) { - logger.warn("buildProtoTagListFromTagMap found null key"); - keyText = "null"; + logger.warn("Conversion from OSM tagmap found null key, skipping..."); + continue; } else { @@ -44,8 +44,8 @@ public List backwardConvert(final Map osmTagMap) if (entry.getValue() == null) { - logger.warn("buildProtoTagListFromTagMap found null value"); - valueText = "null"; + logger.warn("Conversion from OSM tagmap found null value for key {}", keyText); + valueText = ""; } else { From 70abe506a2b90530f7b3b93ad0128146ba82bf69 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 15:03:12 -0700 Subject: [PATCH 13/39] Added an area count to the metadata. --- .../atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index f06aa8bd6d..151b94b4d8 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -69,7 +69,7 @@ public PackedAtlas read(final Resource resource) // initialize atlas metadata final long numberOfPoints = protoAtlasContainer.getNumberOfPoints(); final long numberOfLines = protoAtlasContainer.getNumberOfLines(); - final long numberOfAreas = 0; + final long numberOfAreas = protoAtlasContainer.getNumberOfAreas(); final long numberOfNodes = 0; final long numberOfEdges = 0; final long numberOfRelations = 0; From 1eba0c9539907c0b59fb2c25a56941acd0431337 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 15:11:17 -0700 Subject: [PATCH 14/39] Last of the copy paste shenanigans that I messed up. I hope. lol --- .../geography/atlas/builder/proto/ProtoAtlasBuilder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 151b94b4d8..e65469cc63 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -159,7 +159,7 @@ private void parsePoints(final PackedAtlasBuilder builder, final List tags = area.getTags(); protoAreaBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); - numberOfLines++; + numberOfAreas++; protoAtlasBuilder.addAreas(protoAreaBuilder.build()); } - protoAtlasBuilder.setNumberOfLines(numberOfLines); + protoAtlasBuilder.setNumberOfLines(numberOfAreas); } private void writeLinesToBuilder(final Atlas atlas, From d08f1a8e3a795af37715edd19289a3399b0c6049 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 16:01:28 -0700 Subject: [PATCH 15/39] Ok. If this is not the last copy-paste bug then I resign. --- .../atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index e65469cc63..0244aef2ff 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -178,7 +178,7 @@ private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuil numberOfAreas++; protoAtlasBuilder.addAreas(protoAreaBuilder.build()); } - protoAtlasBuilder.setNumberOfLines(numberOfAreas); + protoAtlasBuilder.setNumberOfAreas(numberOfAreas); } private void writeLinesToBuilder(final Atlas atlas, From 5aa04a8807e36a5e7755de7b98802e133a096799 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 22 Mar 2018 16:02:08 -0700 Subject: [PATCH 16/39] Initial drafts of Node and Edge protos. --- src/main/proto/AtlasContainer.proto | 2 +- src/main/proto/Edge.proto | 18 ++++++++++++ src/main/proto/Node.proto | 19 ++++++++++++ .../builder/proto/ProtoAtlasBuilderTest.java | 29 +++++++++++++++---- 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 src/main/proto/Edge.proto create mode 100644 src/main/proto/Node.proto diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto index 68ad1f3430..20130c92e6 100644 --- a/src/main/proto/AtlasContainer.proto +++ b/src/main/proto/AtlasContainer.proto @@ -18,4 +18,4 @@ message ProtoAtlasContainer { optional int64 numberOfAreas = 5; repeated ProtoArea areas = 6; -} \ No newline at end of file +} diff --git a/src/main/proto/Edge.proto b/src/main/proto/Edge.proto new file mode 100644 index 0000000000..3df4f73266 --- /dev/null +++ b/src/main/proto/Edge.proto @@ -0,0 +1,18 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoEdgeWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Node.proto"; +import "Tag.proto"; + +message ProtoEdge { + optional int64 id = 1; + + optional ProtoNode startNode = 2; + optional ProtoNode endNode = 3; + + repeated ProtoTag tags = 4; +} diff --git a/src/main/proto/Node.proto b/src/main/proto/Node.proto new file mode 100644 index 0000000000..3ff1308749 --- /dev/null +++ b/src/main/proto/Node.proto @@ -0,0 +1,19 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoNodeWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Location.proto"; +import "Tag.proto"; + +message ProtoNode { + optional int64 id = 1; + optional ProtoLocation location = 2; + + repeated int64 inEdgeIds = 3; + repeated int64 outEdgeIds = 4; + + repeated ProtoTag tags = 5; +} diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index 8d998cea97..f8721ae651 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -24,6 +24,14 @@ public class ProtoAtlasBuilderTest { private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilderTest.class); + private static long idCounter = 0L; + + private static long getNextId() + { + idCounter++; + return idCounter; + } + @Test public void testReadWriteConsistency() { @@ -48,21 +56,23 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() final Map tags = new HashMap<>(); final List shapePoints = new ArrayList<>(); + // add points tags.put("building", "yes"); tags.put("name", "eiffel_tower"); - packedAtlasBuilder.addPoint(0, Location.EIFFEL_TOWER, tags); - + packedAtlasBuilder.addPoint(getNextId(), Location.EIFFEL_TOWER, tags); tags.clear(); tags.put("building", "yes"); tags.put("name", "colosseum"); - packedAtlasBuilder.addPoint(1, Location.COLOSSEUM, tags); + packedAtlasBuilder.addPoint(getNextId(), Location.COLOSSEUM, tags); + // add lines tags.clear(); tags.put("path", "yes"); shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); - packedAtlasBuilder.addLine(2, new PolyLine(shapePoints), tags); + packedAtlasBuilder.addLine(getNextId(), new PolyLine(shapePoints), tags); + // add areas tags.clear(); tags.put("triangle", "yes"); tags.put("size", "stupidbig"); @@ -70,8 +80,17 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() shapePoints.add(Location.EIFFEL_TOWER); shapePoints.add(Location.COLOSSEUM); shapePoints.add(Location.STEVENS_CREEK); + packedAtlasBuilder.addArea(getNextId(), new Polygon(shapePoints), tags); + + // add nodes + // tags.clear(); + // tags.put("sometag:namespace", "somevalue"); + // packedAtlasBuilder.addNode(getNextId(), Location.forString("48.3406719,10.5563445"), + // tags); + // tags.clear(); + // packedAtlasBuilder.addNode(getNextId(), Location.forString("48.34204,10.55844"), tags); - packedAtlasBuilder.addArea(3, new Polygon(shapePoints), tags); + // add edges return packedAtlasBuilder; } From 539337e786172efd8777f6cf3a20610a1ab0797d Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Fri, 23 Mar 2018 10:26:21 -0700 Subject: [PATCH 17/39] Added unit tests for the proto converter classes. --- .../ProtoLocationConverterTest.java | 48 ++++++++++++++++++ .../conversion/ProtoTagListConverterTest.java | 50 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java create mode 100644 src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java diff --git a/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java b/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java new file mode 100644 index 0000000000..718a427d07 --- /dev/null +++ b/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java @@ -0,0 +1,48 @@ +package org.openstreetmap.atlas.utilities.conversion; + +import org.junit.Assert; +import org.junit.Test; +import org.openstreetmap.atlas.geography.Latitude; +import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.Longitude; +import org.openstreetmap.atlas.proto.ProtoLocation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author lcram + */ +public class ProtoLocationConverterTest +{ + private static final Logger logger = LoggerFactory.getLogger(ProtoLocationConverterTest.class); + + @Test + public void testLocationToProtoLocation() + { + final ProtoLocationConverter converter = new ProtoLocationConverter(); + final Location location = new Location(Latitude.dm7(1), Longitude.dm7(1)); + + final ProtoLocation protoLocation = ProtoLocation.newBuilder().setLatitude(1) + .setLongitude(1).build(); + final ProtoLocation convertedFromLocation = converter.backwardConvert(location); + + logger.info("{}", protoLocation); + logger.info("{}", convertedFromLocation); + Assert.assertEquals(protoLocation, convertedFromLocation); + } + + @Test + public void testProtoLocationToLocation() + { + final ProtoLocationConverter converter = new ProtoLocationConverter(); + final ProtoLocation protoLocation = ProtoLocation.newBuilder().setLatitude(1) + .setLongitude(1).build(); + + final Location location = new Location(Latitude.dm7(1), Longitude.dm7(1)); + final Location convertedFromProtoLocation = converter.convert(protoLocation); + + logger.info("{}", location); + logger.info("{}", convertedFromProtoLocation); + Assert.assertEquals(location, convertedFromProtoLocation); + } +} diff --git a/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java b/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java new file mode 100644 index 0000000000..1e83cb12bf --- /dev/null +++ b/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java @@ -0,0 +1,50 @@ +package org.openstreetmap.atlas.utilities.conversion; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; +import org.openstreetmap.atlas.proto.ProtoTag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author lcram + */ +public class ProtoTagListConverterTest +{ + private static final Logger logger = LoggerFactory.getLogger(ProtoTagListConverterTest.class); + + @Test + public void testOSMToProtoTagList() + { + final ProtoTagListConverter converter = new ProtoTagListConverter(); + final Map osmTagMap = new HashMap<>(); + osmTagMap.put("key1", "value1"); + osmTagMap.put("key2", "value2"); + final List protoTagList = new ArrayList<>(); + protoTagList.add(ProtoTag.newBuilder().setTagText("key1=value1").build()); + protoTagList.add(ProtoTag.newBuilder().setTagText("key2=value2").build()); + + final List listFromMap = converter.backwardConvert(osmTagMap); + Assert.assertEquals(protoTagList, listFromMap); + } + + @Test + public void testProtoTagListToOSM() + { + final ProtoTagListConverter converter = new ProtoTagListConverter(); + final List protoTagList = new ArrayList<>(); + protoTagList.add(ProtoTag.newBuilder().setTagText("key1=value1").build()); + protoTagList.add(ProtoTag.newBuilder().setTagText("key2=value2").build()); + final Map osmTagMap = new HashMap<>(); + osmTagMap.put("key1", "value1"); + osmTagMap.put("key2", "value2"); + + final Map mapFromList = converter.convert(protoTagList); + Assert.assertEquals(osmTagMap, mapFromList); + } +} From 0bfec87c6d877223a9ce4d025a28ce6fb42d79d2 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Fri, 23 Mar 2018 11:43:04 -0700 Subject: [PATCH 18/39] Changed packages on the converter classes per PR suggestions. --- .../converters/proto}/ProtoLocationConverter.java | 3 ++- .../converters/proto}/ProtoTagListConverter.java | 3 ++- .../converters/proto}/ProtoLocationConverterTest.java | 2 +- .../converters/proto}/ProtoTagListConverterTest.java | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) rename src/main/java/org/openstreetmap/atlas/{utilities/conversion => geography/converters/proto}/ProtoLocationConverter.java (89%) rename src/main/java/org/openstreetmap/atlas/{utilities/conversion => geography/converters/proto}/ProtoTagListConverter.java (95%) rename src/test/java/org/openstreetmap/atlas/{utilities/conversion => geography/converters/proto}/ProtoLocationConverterTest.java (96%) rename src/test/java/org/openstreetmap/atlas/{utilities/conversion => geography/converters/proto}/ProtoTagListConverterTest.java (96%) diff --git a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoLocationConverter.java similarity index 89% rename from src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java rename to src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoLocationConverter.java index f7c548d658..58f19b9c2b 100644 --- a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverter.java +++ b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoLocationConverter.java @@ -1,9 +1,10 @@ -package org.openstreetmap.atlas.utilities.conversion; +package org.openstreetmap.atlas.geography.converters.proto; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.proto.ProtoLocation; +import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; /** * Converts back and forth between ProtoLocation and Location diff --git a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java similarity index 95% rename from src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java rename to src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java index 65cf5fbba1..1d9ec0e255 100644 --- a/src/main/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverter.java +++ b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java @@ -1,4 +1,4 @@ -package org.openstreetmap.atlas.utilities.conversion; +package org.openstreetmap.atlas.geography.converters.proto; import java.util.ArrayList; import java.util.List; @@ -8,6 +8,7 @@ import org.openstreetmap.atlas.proto.ProtoTag; import org.openstreetmap.atlas.utilities.collections.Maps; import org.openstreetmap.atlas.utilities.collections.StringList; +import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoLocationConverterTest.java similarity index 96% rename from src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java rename to src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoLocationConverterTest.java index 718a427d07..6d5029f2a6 100644 --- a/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoLocationConverterTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoLocationConverterTest.java @@ -1,4 +1,4 @@ -package org.openstreetmap.atlas.utilities.conversion; +package org.openstreetmap.atlas.geography.converters.proto; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java similarity index 96% rename from src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java rename to src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java index 1e83cb12bf..e36dfb1d90 100644 --- a/src/test/java/org/openstreetmap/atlas/utilities/conversion/ProtoTagListConverterTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java @@ -1,4 +1,4 @@ -package org.openstreetmap.atlas.utilities.conversion; +package org.openstreetmap.atlas.geography.converters.proto; import java.util.ArrayList; import java.util.HashMap; From 5ce39d57de76e2fcf7e2dbcee8bc1828c716c40f Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Fri, 23 Mar 2018 11:55:22 -0700 Subject: [PATCH 19/39] Forgot to stage this file. --- .../geography/atlas/builder/proto/ProtoAtlasBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 0244aef2ff..b2fd6dfc03 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -18,6 +18,8 @@ import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; +import org.openstreetmap.atlas.geography.converters.proto.ProtoLocationConverter; +import org.openstreetmap.atlas.geography.converters.proto.ProtoTagListConverter; import org.openstreetmap.atlas.proto.ProtoArea; import org.openstreetmap.atlas.proto.ProtoAtlasContainer; import org.openstreetmap.atlas.proto.ProtoAtlasContainer.Builder; @@ -27,8 +29,6 @@ import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Maps; -import org.openstreetmap.atlas.utilities.conversion.ProtoLocationConverter; -import org.openstreetmap.atlas.utilities.conversion.ProtoTagListConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 6359a6f208322bf236ca5c5420b9d6c286fe6f51 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Fri, 23 Mar 2018 14:14:51 -0700 Subject: [PATCH 20/39] Implemented support for Nodes and Edges. --- .../builder/proto/ProtoAtlasBuilder.java | 97 ++++++++++++++++++- src/main/proto/AtlasContainer.proto | 8 ++ src/main/proto/Edge.proto | 9 +- src/main/proto/Node.proto | 4 - .../builder/proto/ProtoAtlasBuilderTest.java | 17 ++-- 5 files changed, 115 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index b2fd6dfc03..d778e69a10 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -14,7 +14,9 @@ import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.items.Area; +import org.openstreetmap.atlas.geography.atlas.items.Edge; import org.openstreetmap.atlas.geography.atlas.items.Line; +import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; @@ -23,8 +25,10 @@ import org.openstreetmap.atlas.proto.ProtoArea; import org.openstreetmap.atlas.proto.ProtoAtlasContainer; import org.openstreetmap.atlas.proto.ProtoAtlasContainer.Builder; +import org.openstreetmap.atlas.proto.ProtoEdge; import org.openstreetmap.atlas.proto.ProtoLine; import org.openstreetmap.atlas.proto.ProtoLocation; +import org.openstreetmap.atlas.proto.ProtoNode; import org.openstreetmap.atlas.proto.ProtoPoint; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; @@ -42,6 +46,9 @@ */ public class ProtoAtlasBuilder { + // File extension for naive protoatlas format + public static final String SUGGESTED_FILE_EXTENSION = ".npatlas"; + private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilder.class); /** @@ -70,8 +77,8 @@ public PackedAtlas read(final Resource resource) final long numberOfPoints = protoAtlasContainer.getNumberOfPoints(); final long numberOfLines = protoAtlasContainer.getNumberOfLines(); final long numberOfAreas = protoAtlasContainer.getNumberOfAreas(); - final long numberOfNodes = 0; - final long numberOfEdges = 0; + final long numberOfNodes = protoAtlasContainer.getNumberOfNodes(); + final long numberOfEdges = protoAtlasContainer.getNumberOfEdges(); final long numberOfRelations = 0; final AtlasSize atlasSize = new AtlasSize(numberOfEdges, numberOfNodes, numberOfAreas, numberOfLines, numberOfPoints, numberOfRelations); @@ -85,6 +92,8 @@ public PackedAtlas read(final Resource resource) parsePoints(builder, protoAtlasContainer.getPointsList()); parseLines(builder, protoAtlasContainer.getLinesList()); parseAreas(builder, protoAtlasContainer.getAreasList()); + parseNodes(builder, protoAtlasContainer.getNodesList()); + parseEdges(builder, protoAtlasContainer.getEdgesList()); return (PackedAtlas) builder.get(); } @@ -101,9 +110,12 @@ public void write(final Atlas atlas, final WritableResource resource) { final ProtoAtlasContainer.Builder protoAtlasBuilder = ProtoAtlasContainer.newBuilder(); + // get the Atlas features into the ProtoAtlasBuilder writePointsToBuilder(atlas, protoAtlasBuilder); writeLinesToBuilder(atlas, protoAtlasBuilder); writeAreasToBuilder(atlas, protoAtlasBuilder); + writeNodesToBuilder(atlas, protoAtlasBuilder); + writeEdgesToBuilder(atlas, protoAtlasBuilder); final ProtoAtlasContainer protoAtlas = protoAtlasBuilder.build(); resource.writeAndClose(protoAtlas.toByteArray()); @@ -128,6 +140,21 @@ private void parseAreas(final PackedAtlasBuilder builder, final List }); } + private void parseEdges(final PackedAtlasBuilder builder, final List edges) + { + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + edges.forEach(protoEdge -> + { + final long identifier = protoEdge.getId(); + final List shapePoints = protoEdge.getShapePointsList().stream() + .map(protoLocationConverter::convert).collect(Collectors.toList()); + final PolyLine geometry = new PolyLine(shapePoints); + final Map tags = protoTagListConverter.convert(protoEdge.getTagsList()); + builder.addEdge(identifier, geometry, tags); + }); + } + private void parseLines(final PackedAtlasBuilder builder, final List lines) { final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); @@ -143,16 +170,31 @@ private void parseLines(final PackedAtlasBuilder builder, final List }); } + private void parseNodes(final PackedAtlasBuilder builder, final List nodes) + { + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + nodes.forEach(protoNode -> + { + final long identifier = protoNode.getId(); + final Longitude longitude = Longitude.dm7(protoNode.getLocation().getLongitude()); + final Latitude latitude = Latitude.dm7(protoNode.getLocation().getLatitude()); + final Location geometry = new Location(latitude, longitude); + final Map tags = protoTagListConverter.convert(protoNode.getTagsList()); + builder.addNode(identifier, geometry, tags); + }); + } + private void parsePoints(final PackedAtlasBuilder builder, final List points) { - final ProtoTagListConverter converter = new ProtoTagListConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); points.forEach(protoPoint -> { final long identifier = protoPoint.getId(); final Longitude longitude = Longitude.dm7(protoPoint.getLocation().getLongitude()); final Latitude latitude = Latitude.dm7(protoPoint.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); - final Map tags = converter.convert(protoPoint.getTagsList()); + final Map tags = protoTagListConverter + .convert(protoPoint.getTagsList()); builder.addPoint(identifier, geometry, tags); }); } @@ -181,6 +223,30 @@ private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuil protoAtlasBuilder.setNumberOfAreas(numberOfAreas); } + private void writeEdgesToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) + { + long numberOfEdges = 0; + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + + for (final Edge edge : atlas.edges()) + { + final ProtoEdge.Builder protoEdgeBuilder = ProtoEdge.newBuilder(); + protoEdgeBuilder.setId(edge.getIdentifier()); + + final List protoLocations = edge.asPolyLine().stream() + .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); + protoEdgeBuilder.addAllShapePoints(protoLocations); + + final Map tags = edge.getTags(); + protoEdgeBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + + numberOfEdges++; + protoAtlasBuilder.addEdges(protoEdgeBuilder.build()); + } + protoAtlasBuilder.setNumberOfEdges(numberOfEdges); + } + private void writeLinesToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { @@ -206,6 +272,29 @@ private void writeLinesToBuilder(final Atlas atlas, protoAtlasBuilder.setNumberOfLines(numberOfLines); } + private void writeNodesToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) + { + long numberOfNodes = 0; + final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + + for (final Node node : atlas.nodes()) + { + final ProtoNode.Builder protoNodeBuilder = ProtoNode.newBuilder(); + + protoNodeBuilder.setId(node.getIdentifier()); + protoNodeBuilder + .setLocation(protoLocationConverter.backwardConvert(node.getLocation())); + + final Map tags = node.getTags(); + protoNodeBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + + numberOfNodes++; + protoAtlasBuilder.addNodes(protoNodeBuilder.build()); + } + protoAtlasBuilder.setNumberOfNodes(numberOfNodes); + } + private void writePointsToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto index 20130c92e6..f2e209b358 100644 --- a/src/main/proto/AtlasContainer.proto +++ b/src/main/proto/AtlasContainer.proto @@ -8,6 +8,8 @@ package org.openstreetmap.atlas.proto; import "Point.proto"; import "Line.proto"; import "Area.proto"; +import "Node.proto"; +import "Edge.proto"; message ProtoAtlasContainer { optional int64 numberOfPoints = 1; @@ -18,4 +20,10 @@ message ProtoAtlasContainer { optional int64 numberOfAreas = 5; repeated ProtoArea areas = 6; + + optional int64 numberOfNodes = 7; + repeated ProtoNode nodes = 8; + + optional int64 numberOfEdges = 9; + repeated ProtoEdge edges = 10; } diff --git a/src/main/proto/Edge.proto b/src/main/proto/Edge.proto index 3df4f73266..352ef28ea3 100644 --- a/src/main/proto/Edge.proto +++ b/src/main/proto/Edge.proto @@ -5,14 +5,11 @@ option java_outer_classname = "ProtoEdgeWrapper"; package org.openstreetmap.atlas.proto; -import "Node.proto"; +import "Location.proto"; import "Tag.proto"; message ProtoEdge { optional int64 id = 1; - - optional ProtoNode startNode = 2; - optional ProtoNode endNode = 3; - - repeated ProtoTag tags = 4; + repeated ProtoLocation shapePoints = 2; + repeated ProtoTag tags = 5; } diff --git a/src/main/proto/Node.proto b/src/main/proto/Node.proto index 3ff1308749..4a8106de2c 100644 --- a/src/main/proto/Node.proto +++ b/src/main/proto/Node.proto @@ -11,9 +11,5 @@ import "Tag.proto"; message ProtoNode { optional int64 id = 1; optional ProtoLocation location = 2; - - repeated int64 inEdgeIds = 3; - repeated int64 outEdgeIds = 4; - repeated ProtoTag tags = 5; } diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index f8721ae651..37696b089a 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -83,14 +83,19 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() packedAtlasBuilder.addArea(getNextId(), new Polygon(shapePoints), tags); // add nodes - // tags.clear(); - // tags.put("sometag:namespace", "somevalue"); - // packedAtlasBuilder.addNode(getNextId(), Location.forString("48.3406719,10.5563445"), - // tags); - // tags.clear(); - // packedAtlasBuilder.addNode(getNextId(), Location.forString("48.34204,10.55844"), tags); + tags.clear(); + tags.put("sometag:namespace", "somevalue"); + packedAtlasBuilder.addNode(getNextId(), Location.forString("48.3406719,10.5563445"), tags); + tags.clear(); + packedAtlasBuilder.addNode(getNextId(), Location.forString("48.34204,10.55844"), tags); // add edges + tags.clear(); + tags.put("edge", "yes"); + shapePoints.clear(); + shapePoints.add(Location.forString("48.3406719,10.5563445")); + shapePoints.add(Location.forString("48.34204,10.55844")); + packedAtlasBuilder.addEdge(getNextId(), new PolyLine(shapePoints), tags); return packedAtlasBuilder; } From ba3b136ffdf1381001ab453b47d2cdd763635963 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Fri, 23 Mar 2018 16:41:23 -0700 Subject: [PATCH 21/39] Almost done with relations. Small bug with the OSM identifier. See unit test. --- .../builder/proto/ProtoAtlasBuilder.java | 82 +++++++++++++++++-- src/main/proto/AtlasContainer.proto | 4 + src/main/proto/Relation.proto | 30 +++++++ .../builder/proto/ProtoAtlasBuilderTest.java | 7 ++ 4 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 src/main/proto/Relation.proto diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index d778e69a10..2d1bae227b 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -13,11 +13,15 @@ import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasMetaData; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; +import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.Edge; +import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.items.Line; import org.openstreetmap.atlas.geography.atlas.items.Node; import org.openstreetmap.atlas.geography.atlas.items.Point; +import org.openstreetmap.atlas.geography.atlas.items.Relation; +import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.geography.converters.proto.ProtoLocationConverter; @@ -30,6 +34,7 @@ import org.openstreetmap.atlas.proto.ProtoLocation; import org.openstreetmap.atlas.proto.ProtoNode; import org.openstreetmap.atlas.proto.ProtoPoint; +import org.openstreetmap.atlas.proto.ProtoRelation; import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Maps; @@ -74,14 +79,10 @@ public PackedAtlas read(final Resource resource) } // initialize atlas metadata - final long numberOfPoints = protoAtlasContainer.getNumberOfPoints(); - final long numberOfLines = protoAtlasContainer.getNumberOfLines(); - final long numberOfAreas = protoAtlasContainer.getNumberOfAreas(); - final long numberOfNodes = protoAtlasContainer.getNumberOfNodes(); - final long numberOfEdges = protoAtlasContainer.getNumberOfEdges(); - final long numberOfRelations = 0; - final AtlasSize atlasSize = new AtlasSize(numberOfEdges, numberOfNodes, numberOfAreas, - numberOfLines, numberOfPoints, numberOfRelations); + final AtlasSize atlasSize = new AtlasSize(protoAtlasContainer.getNumberOfEdges(), + protoAtlasContainer.getNumberOfNodes(), protoAtlasContainer.getNumberOfAreas(), + protoAtlasContainer.getNumberOfLines(), protoAtlasContainer.getNumberOfPoints(), + protoAtlasContainer.getNumberOfRelations()); // TODO it would be nice to programmatically determine which protobuf version is in use final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, true, "unknown", "ProtoAtlas", "unknown", "unknown", Maps.hashMap()); @@ -94,6 +95,7 @@ public PackedAtlas read(final Resource resource) parseAreas(builder, protoAtlasContainer.getAreasList()); parseNodes(builder, protoAtlasContainer.getNodesList()); parseEdges(builder, protoAtlasContainer.getEdgesList()); + parseRelations(builder, protoAtlasContainer.getRelationsList()); return (PackedAtlas) builder.get(); } @@ -110,12 +112,13 @@ public void write(final Atlas atlas, final WritableResource resource) { final ProtoAtlasContainer.Builder protoAtlasBuilder = ProtoAtlasContainer.newBuilder(); - // get the Atlas features into the ProtoAtlasBuilder + // put the Atlas features into the ProtoAtlasBuilder writePointsToBuilder(atlas, protoAtlasBuilder); writeLinesToBuilder(atlas, protoAtlasBuilder); writeAreasToBuilder(atlas, protoAtlasBuilder); writeNodesToBuilder(atlas, protoAtlasBuilder); writeEdgesToBuilder(atlas, protoAtlasBuilder); + writeRelationsToBuilder(atlas, protoAtlasBuilder); final ProtoAtlasContainer protoAtlas = protoAtlasBuilder.build(); resource.writeAndClose(protoAtlas.toByteArray()); @@ -199,6 +202,37 @@ private void parsePoints(final PackedAtlasBuilder builder, final List + { + final long memberId = protoRelationBean.getMemberId(); + final String memberRole = protoRelationBean.getMemberRole(); + final ItemType memberType = ItemType + .forValue(protoRelationBean.getMemberType().getNumber()); + bean.addItem(memberId, memberRole, memberType); + }); + + return bean; + } + + private void parseRelations(final PackedAtlasBuilder builder, + final List relations) + { + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + relations.forEach(protoRelation -> + { + final long identifier = protoRelation.getId(); + final RelationBean bean = parseRelationBean(protoRelation); + final Map tags = protoTagListConverter + .convert(protoRelation.getTagsList()); + // TODO should identifier and OSMIdentifier be the same? + builder.addRelation(identifier, identifier, bean, tags); + }); + } + private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) { long numberOfAreas = 0; @@ -318,4 +352,34 @@ private void writePointsToBuilder(final Atlas atlas, } protoAtlasBuilder.setNumberOfPoints(numberOfPoints); } + + private void writeRelationsToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) + { + long numberOfRelations = 0; + final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + + for (final Relation relation : atlas.relations()) + { + final ProtoRelation.Builder protoRelationBuilder = ProtoRelation.newBuilder(); + protoRelationBuilder.setId(relation.getIdentifier()); + for (final RelationMember member : relation.members()) + { + final ProtoRelation.RelationBean.Builder beanBuilder = ProtoRelation.RelationBean + .newBuilder(); + beanBuilder.setMemberId(member.getRelationIdentifier()); + beanBuilder.setMemberRole(member.getRole()); + final ItemType type = ItemType.forEntity(member.getEntity()); + beanBuilder.setMemberType(ProtoRelation.ProtoItemType.valueOf(type.getValue())); + + // TODO there may be a problem here if there are more relations in this atlas than + // possible int32 values. This is because protobuf internally stores the protobeans + // list as an ArrayList + protoRelationBuilder.addBeans(beanBuilder.build()); + + } + numberOfRelations++; + protoAtlasBuilder.addRelations(protoRelationBuilder.build()); + } + protoAtlasBuilder.setNumberOfRelations(numberOfRelations); + } } diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto index f2e209b358..23a6ff1fd2 100644 --- a/src/main/proto/AtlasContainer.proto +++ b/src/main/proto/AtlasContainer.proto @@ -10,6 +10,7 @@ import "Line.proto"; import "Area.proto"; import "Node.proto"; import "Edge.proto"; +import "Relation.proto"; message ProtoAtlasContainer { optional int64 numberOfPoints = 1; @@ -26,4 +27,7 @@ message ProtoAtlasContainer { optional int64 numberOfEdges = 9; repeated ProtoEdge edges = 10; + + optional int64 numberOfRelations = 11; + repeated ProtoRelation relations = 12; } diff --git a/src/main/proto/Relation.proto b/src/main/proto/Relation.proto new file mode 100644 index 0000000000..d632486d39 --- /dev/null +++ b/src/main/proto/Relation.proto @@ -0,0 +1,30 @@ +syntax = "proto2"; + +option java_multiple_files = true; +option java_outer_classname = "ProtoRelationWrapper"; + +package org.openstreetmap.atlas.proto; + +import "Tag.proto"; + +message ProtoRelation { + optional int64 id = 1; + repeated ProtoTag tags = 2; + + enum ProtoItemType { + NODE = 0; + EDGE = 1; + AREA = 2; + LINE = 3; + POINT = 4; + RELATION = 5; + } + + message RelationBean { + optional int64 memberId = 1; + optional string memberRole = 2; + optional ProtoItemType memberType = 3; + } + + repeated RelationBean beans = 3; +} diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index 37696b089a..7e77dd2c4a 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -97,6 +97,13 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() shapePoints.add(Location.forString("48.34204,10.55844")); packedAtlasBuilder.addEdge(getNextId(), new PolyLine(shapePoints), tags); + // add relations + // tags.clear(); + // tags.put("relationtag", "somevalue"); + // final RelationBean bean = new RelationBean(); + // bean.addItem(1L, "This is the Eiffel Tower Point", ItemType.POINT); + // packedAtlasBuilder.addRelation(getNextId(), idCounter, bean, tags); + return packedAtlasBuilder; } } From 6db5ee74850f80a446dd5c1311bd5c196a372836 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Sat, 24 Mar 2018 09:08:20 -0700 Subject: [PATCH 22/39] Fixed bug with relation packing. Relations work! --- .../atlas/builder/proto/ProtoAtlasBuilder.java | 5 ++++- .../atlas/builder/proto/ProtoAtlasBuilderTest.java | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 2d1bae227b..f9d3e08241 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -366,7 +366,7 @@ private void writeRelationsToBuilder(final Atlas atlas, final Builder protoAtlas { final ProtoRelation.RelationBean.Builder beanBuilder = ProtoRelation.RelationBean .newBuilder(); - beanBuilder.setMemberId(member.getRelationIdentifier()); + beanBuilder.setMemberId(member.getEntity().getIdentifier()); beanBuilder.setMemberRole(member.getRole()); final ItemType type = ItemType.forEntity(member.getEntity()); beanBuilder.setMemberType(ProtoRelation.ProtoItemType.valueOf(type.getValue())); @@ -377,6 +377,9 @@ private void writeRelationsToBuilder(final Atlas atlas, final Builder protoAtlas protoRelationBuilder.addBeans(beanBuilder.build()); } + final Map tags = relation.getTags(); + protoRelationBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + numberOfRelations++; protoAtlasBuilder.addRelations(protoRelationBuilder.build()); } diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index 7e77dd2c4a..38e84a229d 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -11,6 +11,8 @@ import org.openstreetmap.atlas.geography.PolyLine; import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.Atlas; +import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; +import org.openstreetmap.atlas.geography.atlas.items.ItemType; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; import org.openstreetmap.atlas.streaming.resource.ByteArrayResource; import org.openstreetmap.atlas.streaming.resource.WritableResource; @@ -98,11 +100,11 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() packedAtlasBuilder.addEdge(getNextId(), new PolyLine(shapePoints), tags); // add relations - // tags.clear(); - // tags.put("relationtag", "somevalue"); - // final RelationBean bean = new RelationBean(); - // bean.addItem(1L, "This is the Eiffel Tower Point", ItemType.POINT); - // packedAtlasBuilder.addRelation(getNextId(), idCounter, bean, tags); + tags.clear(); + tags.put("relationtag", "somevalue"); + final RelationBean bean = new RelationBean(); + bean.addItem(1L, "This is the Eiffel Tower Point", ItemType.POINT); + packedAtlasBuilder.addRelation(getNextId(), idCounter, bean, tags); return packedAtlasBuilder; } From a7266a6eefc7441e7c081b4a78fb75e947f357f1 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 26 Mar 2018 06:40:27 -0700 Subject: [PATCH 23/39] Lots of ProtoAtlas testing. --- .../atlas/builder/proto/ProtoAtlasBuilderTest.java | 11 ++++++++++- .../proto/ProtoTagListConverterTest.java | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java index 38e84a229d..598895ce09 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilderTest.java @@ -102,10 +102,19 @@ private PackedAtlasBuilder setUpTestAtlasBuilder() // add relations tags.clear(); tags.put("relationtag", "somevalue"); - final RelationBean bean = new RelationBean(); + RelationBean bean = new RelationBean(); bean.addItem(1L, "This is the Eiffel Tower Point", ItemType.POINT); packedAtlasBuilder.addRelation(getNextId(), idCounter, bean, tags); + tags.clear(); + tags.put("name", "coolstuff"); + tags.put("has_subrelation", "yes"); + bean = new RelationBean(); + bean.addItem(1L, "Eiffel Tower", ItemType.POINT); + bean.addItem(2L, "Colosseum", ItemType.POINT); + bean.addItem(8L, "subrelation", ItemType.RELATION); + packedAtlasBuilder.addRelation(getNextId(), idCounter, bean, tags); + return packedAtlasBuilder; } } diff --git a/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java index e36dfb1d90..d2e58f343e 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java @@ -18,6 +18,20 @@ public class ProtoTagListConverterTest { private static final Logger logger = LoggerFactory.getLogger(ProtoTagListConverterTest.class); + @Test + public void testEmptyConversion() + { + final ProtoTagListConverter converter = new ProtoTagListConverter(); + final Map osmTagMap = new HashMap<>(); + final List protoTagList = new ArrayList<>(); + + final List listFromMap = converter.backwardConvert(osmTagMap); + Assert.assertEquals(protoTagList, listFromMap); + + final Map mapFromList = converter.convert(protoTagList); + Assert.assertEquals(osmTagMap, mapFromList); + } + @Test public void testOSMToProtoTagList() { From 27aa4e29142b6c22e09a41c9c378360d959e8de9 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 26 Mar 2018 09:38:58 -0700 Subject: [PATCH 24/39] Added some commands for converting between Packed and ProtoAtlases. There seems to be some bugs. --- .../atlas/geography/atlas/Atlas.java | 8 ++ .../atlas/geography/atlas/BareAtlas.java | 7 ++ .../command/PackedToProtoAtlasSubCommand.java | 101 +++++++++++++++++ .../command/ProtoToPackedAtlasSubCommand.java | 102 ++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java create mode 100644 src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/Atlas.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/Atlas.java index b4b085e3e4..9dcd33e5b6 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/Atlas.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/Atlas.java @@ -698,6 +698,14 @@ Iterable relationsWithEntitiesIntersecting(Polygon polygon, */ void saveAsList(WritableResource resource); + /** + * Save as a naive proto file + * + * @param resource + * The resource to write to + */ + void saveAsProto(WritableResource resource); + /** * Save as a text file * diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/BareAtlas.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/BareAtlas.java index 7805c579b4..6463124242 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/BareAtlas.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/BareAtlas.java @@ -23,6 +23,7 @@ import org.openstreetmap.atlas.geography.Polygon; import org.openstreetmap.atlas.geography.atlas.builder.AtlasSize; import org.openstreetmap.atlas.geography.atlas.builder.RelationBean; +import org.openstreetmap.atlas.geography.atlas.builder.proto.ProtoAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.builder.text.TextAtlasBuilder; import org.openstreetmap.atlas.geography.atlas.items.Area; import org.openstreetmap.atlas.geography.atlas.items.AtlasEntity; @@ -469,6 +470,12 @@ public void saveAsList(final WritableResource resource) } } + @Override + public void saveAsProto(final WritableResource resource) + { + new ProtoAtlasBuilder().write(this, resource); + } + @Override public void saveAsText(final WritableResource resource) { diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java new file mode 100644 index 0000000000..6aa269126e --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java @@ -0,0 +1,101 @@ +package org.openstreetmap.atlas.geography.atlas.command; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.openstreetmap.atlas.exception.CoreException; +import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; +import org.openstreetmap.atlas.streaming.resource.File; +import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; +import org.openstreetmap.atlas.utilities.runtime.Command.Switch; +import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; +import org.openstreetmap.atlas.utilities.runtime.CommandMap; +import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Command for converting a serialized PackedAtlas to a serialized ProtoAtlas + * + * @author lcram + */ +public class PackedToProtoAtlasSubCommand implements FlexibleSubCommand +{ + private static final Logger logger = LoggerFactory.getLogger(PackedToProtoAtlasSubCommand.class); + + private static final String NAME = "packed-to-proto"; + private static final String DESCRIPTION = "converts a packed atlas to a naive proto-based atlas"; + private static final String PACKED_SWITCH_TEXT = "packed-atlas"; + private static final String PROTO_SWITCH_TEXT = "proto-atlas"; + + private static final Switch INPUT_PARAMETER = new Switch<>(PACKED_SWITCH_TEXT, + "Input atlas data in text atlas format", Paths::get, Optionality.REQUIRED); + + private static final Switch OUTPUT_PARAMETER = new Switch<>(PROTO_SWITCH_TEXT, + "Output atlas data path", Paths::get, Optionality.REQUIRED); + + private Path inputPath; + private Path outputPath; + + @Override + public int execute(final CommandMap map) + { + this.inputPath = (Path) map.get(INPUT_PARAMETER); + this.outputPath = (Path) map.get(OUTPUT_PARAMETER); + verifyArguments(); + PackedAtlas.load(new File(this.inputPath.toFile())) + .saveAsProto(new File(this.outputPath.toFile())); + + return 0; + } + + @Override + public String getDescription() + { + return DESCRIPTION; + } + + @Override + public String getName() + { + return NAME; + } + + @Override + public SwitchList switches() + { + return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER); + } + + @Override + public void usage(final PrintStream writer) + { + writer.println("-" + PACKED_SWITCH_TEXT + "=/input/path/to/packed/atlas"); + writer.println("-" + PROTO_SWITCH_TEXT + "=/output/path/to/proto/atlas"); + } + + private void verifyArguments() + { + if (!Files.isRegularFile(this.inputPath)) + { + throw new CoreException("{} is not a readable file", this.inputPath); + } + + try + { + if (Files.isDirectory(this.outputPath)) + { + throw new CoreException("{} is a directory. Aborting", this.outputPath); + } + Files.createDirectories(this.outputPath.getParent()); + } + catch (final IOException exception) + { + throw new CoreException("Error when creating directories {}", + this.outputPath.getParent(), exception); + } + } +} diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java new file mode 100644 index 0000000000..39e722dab2 --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java @@ -0,0 +1,102 @@ +package org.openstreetmap.atlas.geography.atlas.command; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.openstreetmap.atlas.exception.CoreException; +import org.openstreetmap.atlas.geography.atlas.builder.proto.ProtoAtlasBuilder; +import org.openstreetmap.atlas.streaming.resource.File; +import org.openstreetmap.atlas.utilities.runtime.Command.Optionality; +import org.openstreetmap.atlas.utilities.runtime.Command.Switch; +import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; +import org.openstreetmap.atlas.utilities.runtime.CommandMap; +import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Command for converting a serialized ProtoAtlas to a serialized PackedAtlas + * + * @author lcram + */ +public class ProtoToPackedAtlasSubCommand implements FlexibleSubCommand +{ + private static final Logger logger = LoggerFactory + .getLogger(ProtoToPackedAtlasSubCommand.class); + + private static final String NAME = "proto-to-packed"; + private static final String DESCRIPTION = "converts a naive proto-based atlas to a packed atlas"; + private static final String PROTO_SWITCH_TEXT = "proto-atlas"; + private static final String PACKED_SWITCH_TEXT = "packed-atlas"; + + private static final Switch INPUT_PARAMETER = new Switch<>(PROTO_SWITCH_TEXT, + "Input atlas data in text atlas format", Paths::get, Optionality.REQUIRED); + + private static final Switch OUTPUT_PARAMETER = new Switch<>(PACKED_SWITCH_TEXT, + "Output atlas data path", Paths::get, Optionality.REQUIRED); + + private Path inputPath; + private Path outputPath; + + @Override + public int execute(final CommandMap map) + { + this.inputPath = (Path) map.get(INPUT_PARAMETER); + this.outputPath = (Path) map.get(OUTPUT_PARAMETER); + verifyArguments(); + new ProtoAtlasBuilder().read(new File(this.inputPath.toFile())) + .save(new File(this.outputPath.toFile())); + + return 0; + } + + @Override + public String getDescription() + { + return DESCRIPTION; + } + + @Override + public String getName() + { + return NAME; + } + + @Override + public SwitchList switches() + { + return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER); + } + + @Override + public void usage(final PrintStream writer) + { + writer.println("-" + PROTO_SWITCH_TEXT + "=/input/path/to/proto/atlas"); + writer.println("-" + PACKED_SWITCH_TEXT + "=/output/path/to/packed/atlas"); + } + + private void verifyArguments() + { + if (!Files.isRegularFile(this.inputPath)) + { + throw new CoreException("{} is not a readable file", this.inputPath); + } + + try + { + if (Files.isDirectory(this.outputPath)) + { + throw new CoreException("{} is a directory. Aborting", this.outputPath); + } + Files.createDirectories(this.outputPath.getParent()); + } + catch (final IOException exception) + { + throw new CoreException("Error when creating directories {}", + this.outputPath.getParent(), exception); + } + } +} From 4cd48d041163571a242374900f65540f226a86cb Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 26 Mar 2018 10:04:57 -0700 Subject: [PATCH 25/39] Fixed spotless formatting issue. --- .../geography/atlas/command/PackedToProtoAtlasSubCommand.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java index 6aa269126e..4afa6fd8d3 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java @@ -24,7 +24,8 @@ */ public class PackedToProtoAtlasSubCommand implements FlexibleSubCommand { - private static final Logger logger = LoggerFactory.getLogger(PackedToProtoAtlasSubCommand.class); + private static final Logger logger = LoggerFactory + .getLogger(PackedToProtoAtlasSubCommand.class); private static final String NAME = "packed-to-proto"; private static final String DESCRIPTION = "converts a packed atlas to a naive proto-based atlas"; From 4a06a5dccfb8c9702410d128848118e5e28ddfb3 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 26 Mar 2018 10:08:33 -0700 Subject: [PATCH 26/39] Wrote a quick test class to investigate the proto size discrepancies. --- .../AtlasConversionTestingSubCommand.java | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java new file mode 100644 index 0000000000..caf43b43dd --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java @@ -0,0 +1,134 @@ +package org.openstreetmap.atlas.geography.atlas.command; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.openstreetmap.atlas.geography.Location; +import org.openstreetmap.atlas.geography.atlas.Atlas; +import org.openstreetmap.atlas.geography.atlas.builder.proto.ProtoAtlasBuilder; +import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; +import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; +import org.openstreetmap.atlas.streaming.resource.File; +import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; +import org.openstreetmap.atlas.utilities.runtime.CommandMap; +import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; + +/** + * @author lcram + */ +public class AtlasConversionTestingSubCommand implements FlexibleSubCommand +{ + + private static final String NAME = "atlas-conversion-testing"; + private static final String DESCRIPTION = "hardcoded class for testing atlas conversions"; + + @Override + public int execute(final CommandMap map) + { + final PackedAtlasBuilder packedAtlasBuilder = new PackedAtlasBuilder(); + final Map tags = new HashMap<>(); + final List shapePoints = new ArrayList<>(); + + // add points + tags.put("building", "yes"); + tags.put("name", "eiffel_tower"); + packedAtlasBuilder.addPoint(1, Location.EIFFEL_TOWER, tags); + tags.clear(); + tags.put("building", "yes"); + tags.put("name", "colosseum"); + packedAtlasBuilder.addPoint(2, Location.COLOSSEUM, tags); + + // // add lines + // tags.clear(); + // tags.put("path", "yes"); + // shapePoints.add(Location.EIFFEL_TOWER); + // shapePoints.add(Location.COLOSSEUM); + // packedAtlasBuilder.addLine(3, new PolyLine(shapePoints), tags); + // + // // add areas + // tags.clear(); + // tags.put("triangle", "yes"); + // tags.put("size", "stupidbig"); + // shapePoints.clear(); + // shapePoints.add(Location.EIFFEL_TOWER); + // shapePoints.add(Location.COLOSSEUM); + // shapePoints.add(Location.STEVENS_CREEK); + // packedAtlasBuilder.addArea(4, new Polygon(shapePoints), tags); + // + // // add nodes + // tags.clear(); + // tags.put("sometag:namespace", "somevalue"); + // packedAtlasBuilder.addNode(5, Location.forString("48.3406719,10.5563445"), tags); + // tags.clear(); + // packedAtlasBuilder.addNode(6, Location.forString("48.34204,10.55844"), tags); + // + // // add edges + // tags.clear(); + // tags.put("edge", "yes"); + // shapePoints.clear(); + // shapePoints.add(Location.forString("48.3406719,10.5563445")); + // shapePoints.add(Location.forString("48.34204,10.55844")); + // packedAtlasBuilder.addEdge(7, new PolyLine(shapePoints), tags); + // + // // add relations + // tags.clear(); + // tags.put("relationtag", "somevalue"); + // RelationBean bean = new RelationBean(); + // bean.addItem(1L, "This is the Eiffel Tower Point", ItemType.POINT); + // packedAtlasBuilder.addRelation(8, 8, bean, tags); + // + // tags.clear(); + // tags.put("name", "coolstuff"); + // tags.put("has_subrelation", "yes"); + // bean = new RelationBean(); + // bean.addItem(1L, "Eiffel Tower", ItemType.POINT); + // bean.addItem(2L, "Colosseum", ItemType.POINT); + // bean.addItem(8L, "subrelation", ItemType.RELATION); + // packedAtlasBuilder.addRelation(9, 9, bean, tags); + + final Atlas atlas = packedAtlasBuilder.get(); + atlas.save(new File("/Users/lucascram/Desktop/test.atlas")); + + PackedAtlas.load(new File("/Users/lucascram/Desktop/test.atlas")) + .saveAsProto(new File("/Users/lucascram/Desktop/test.npatlas")); + + new ProtoAtlasBuilder().read(new File("/Users/lucascram/Desktop/test.npatlas")) + .save(new File("/Users/lucascram/Desktop/test_out.atlas")); + + PackedAtlas.load(new File("/Users/lucascram/Desktop/test.atlas")) + .saveAsText(new File("/Users/lucascram/Desktop/test.textatlas")); + + PackedAtlas.load(new File("/Users/lucascram/Desktop/test_out.atlas")) + .saveAsText(new File("/Users/lucascram/Desktop/test_out.textatlas")); + + return 0; + } + + @Override + public String getDescription() + { + return DESCRIPTION; + } + + @Override + public String getName() + { + return NAME; + } + + @Override + public SwitchList switches() + { + return new SwitchList(); + } + + @Override + public void usage(final PrintStream writer) + { + + } + +} From cd9fb958987bb6f2edece621ec59e81ccb98def4 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 26 Mar 2018 14:26:14 -0700 Subject: [PATCH 27/39] Fixed bug where tags containing delimiter characters were not deserialized correctly. --- .../builder/proto/ProtoAtlasBuilder.java | 2 + .../AtlasConversionTestingSubCommand.java | 134 ------------------ .../proto/ProtoTagListConverter.java | 15 +- src/main/proto/Tag.proto | 3 +- .../proto/ProtoTagListConverterTest.java | 8 +- 5 files changed, 11 insertions(+), 151 deletions(-) delete mode 100644 src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index f9d3e08241..fc962191f8 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -83,9 +83,11 @@ public PackedAtlas read(final Resource resource) protoAtlasContainer.getNumberOfNodes(), protoAtlasContainer.getNumberOfAreas(), protoAtlasContainer.getNumberOfLines(), protoAtlasContainer.getNumberOfPoints(), protoAtlasContainer.getNumberOfRelations()); + // TODO it would be nice to programmatically determine which protobuf version is in use final AtlasMetaData atlasMetaData = new AtlasMetaData(atlasSize, true, "unknown", "ProtoAtlas", "unknown", "unknown", Maps.hashMap()); + final PackedAtlasBuilder builder = new PackedAtlasBuilder().withSizeEstimates(atlasSize) .withMetaData(atlasMetaData).withName(resource.getName()); diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java deleted file mode 100644 index caf43b43dd..0000000000 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/AtlasConversionTestingSubCommand.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.openstreetmap.atlas.geography.atlas.command; - -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.openstreetmap.atlas.geography.Location; -import org.openstreetmap.atlas.geography.atlas.Atlas; -import org.openstreetmap.atlas.geography.atlas.builder.proto.ProtoAtlasBuilder; -import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; -import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; -import org.openstreetmap.atlas.streaming.resource.File; -import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; -import org.openstreetmap.atlas.utilities.runtime.CommandMap; -import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; - -/** - * @author lcram - */ -public class AtlasConversionTestingSubCommand implements FlexibleSubCommand -{ - - private static final String NAME = "atlas-conversion-testing"; - private static final String DESCRIPTION = "hardcoded class for testing atlas conversions"; - - @Override - public int execute(final CommandMap map) - { - final PackedAtlasBuilder packedAtlasBuilder = new PackedAtlasBuilder(); - final Map tags = new HashMap<>(); - final List shapePoints = new ArrayList<>(); - - // add points - tags.put("building", "yes"); - tags.put("name", "eiffel_tower"); - packedAtlasBuilder.addPoint(1, Location.EIFFEL_TOWER, tags); - tags.clear(); - tags.put("building", "yes"); - tags.put("name", "colosseum"); - packedAtlasBuilder.addPoint(2, Location.COLOSSEUM, tags); - - // // add lines - // tags.clear(); - // tags.put("path", "yes"); - // shapePoints.add(Location.EIFFEL_TOWER); - // shapePoints.add(Location.COLOSSEUM); - // packedAtlasBuilder.addLine(3, new PolyLine(shapePoints), tags); - // - // // add areas - // tags.clear(); - // tags.put("triangle", "yes"); - // tags.put("size", "stupidbig"); - // shapePoints.clear(); - // shapePoints.add(Location.EIFFEL_TOWER); - // shapePoints.add(Location.COLOSSEUM); - // shapePoints.add(Location.STEVENS_CREEK); - // packedAtlasBuilder.addArea(4, new Polygon(shapePoints), tags); - // - // // add nodes - // tags.clear(); - // tags.put("sometag:namespace", "somevalue"); - // packedAtlasBuilder.addNode(5, Location.forString("48.3406719,10.5563445"), tags); - // tags.clear(); - // packedAtlasBuilder.addNode(6, Location.forString("48.34204,10.55844"), tags); - // - // // add edges - // tags.clear(); - // tags.put("edge", "yes"); - // shapePoints.clear(); - // shapePoints.add(Location.forString("48.3406719,10.5563445")); - // shapePoints.add(Location.forString("48.34204,10.55844")); - // packedAtlasBuilder.addEdge(7, new PolyLine(shapePoints), tags); - // - // // add relations - // tags.clear(); - // tags.put("relationtag", "somevalue"); - // RelationBean bean = new RelationBean(); - // bean.addItem(1L, "This is the Eiffel Tower Point", ItemType.POINT); - // packedAtlasBuilder.addRelation(8, 8, bean, tags); - // - // tags.clear(); - // tags.put("name", "coolstuff"); - // tags.put("has_subrelation", "yes"); - // bean = new RelationBean(); - // bean.addItem(1L, "Eiffel Tower", ItemType.POINT); - // bean.addItem(2L, "Colosseum", ItemType.POINT); - // bean.addItem(8L, "subrelation", ItemType.RELATION); - // packedAtlasBuilder.addRelation(9, 9, bean, tags); - - final Atlas atlas = packedAtlasBuilder.get(); - atlas.save(new File("/Users/lucascram/Desktop/test.atlas")); - - PackedAtlas.load(new File("/Users/lucascram/Desktop/test.atlas")) - .saveAsProto(new File("/Users/lucascram/Desktop/test.npatlas")); - - new ProtoAtlasBuilder().read(new File("/Users/lucascram/Desktop/test.npatlas")) - .save(new File("/Users/lucascram/Desktop/test_out.atlas")); - - PackedAtlas.load(new File("/Users/lucascram/Desktop/test.atlas")) - .saveAsText(new File("/Users/lucascram/Desktop/test.textatlas")); - - PackedAtlas.load(new File("/Users/lucascram/Desktop/test_out.atlas")) - .saveAsText(new File("/Users/lucascram/Desktop/test_out.textatlas")); - - return 0; - } - - @Override - public String getDescription() - { - return DESCRIPTION; - } - - @Override - public String getName() - { - return NAME; - } - - @Override - public SwitchList switches() - { - return new SwitchList(); - } - - @Override - public void usage(final PrintStream writer) - { - - } - -} diff --git a/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java index 1d9ec0e255..8362c528ba 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java +++ b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java @@ -7,7 +7,6 @@ import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.proto.ProtoTag; import org.openstreetmap.atlas.utilities.collections.Maps; -import org.openstreetmap.atlas.utilities.collections.StringList; import org.openstreetmap.atlas.utilities.conversion.TwoWayConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,8 +52,8 @@ public List backwardConvert(final Map osmTagMap) valueText = entry.getValue(); } - final String fullTag = keyText + TAG_DELIMITER + valueText; - tagBuilder.setTagText(fullTag); + tagBuilder.setKey(keyText); + tagBuilder.setValue(valueText); protoTags.add(tagBuilder.build()); } @@ -69,15 +68,7 @@ public Map convert(final List protoTagList) final Map result = Maps.hashMap(); for (final ProtoTag tag : protoTagList) { - final StringList tagSplit = StringList.split(tag.getTagText(), TAG_DELIMITER); - if (tagSplit.size() == 2) - { - result.put(tagSplit.get(0), tagSplit.get(1)); - } - if (tagSplit.size() == 1) - { - result.put(tagSplit.get(0), ""); - } + result.put(tag.getKey(), tag.getValue()); } return result; } diff --git a/src/main/proto/Tag.proto b/src/main/proto/Tag.proto index 19bea42ed7..3c0c8301ab 100644 --- a/src/main/proto/Tag.proto +++ b/src/main/proto/Tag.proto @@ -6,5 +6,6 @@ option java_outer_classname = "ProtoTagWrapper"; package org.openstreetmap.atlas.proto; message ProtoTag { - optional string tagText = 1; + optional string key = 1; + optional string value = 2; } diff --git a/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java index d2e58f343e..1925aa2420 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java @@ -40,8 +40,8 @@ public void testOSMToProtoTagList() osmTagMap.put("key1", "value1"); osmTagMap.put("key2", "value2"); final List protoTagList = new ArrayList<>(); - protoTagList.add(ProtoTag.newBuilder().setTagText("key1=value1").build()); - protoTagList.add(ProtoTag.newBuilder().setTagText("key2=value2").build()); + protoTagList.add(ProtoTag.newBuilder().setKey("key1").setValue("value1").build()); + protoTagList.add(ProtoTag.newBuilder().setKey("key2").setValue("value2").build()); final List listFromMap = converter.backwardConvert(osmTagMap); Assert.assertEquals(protoTagList, listFromMap); @@ -52,8 +52,8 @@ public void testProtoTagListToOSM() { final ProtoTagListConverter converter = new ProtoTagListConverter(); final List protoTagList = new ArrayList<>(); - protoTagList.add(ProtoTag.newBuilder().setTagText("key1=value1").build()); - protoTagList.add(ProtoTag.newBuilder().setTagText("key2=value2").build()); + protoTagList.add(ProtoTag.newBuilder().setKey("key1").setValue("value1").build()); + protoTagList.add(ProtoTag.newBuilder().setKey("key2").setValue("value2").build()); final Map osmTagMap = new HashMap<>(); osmTagMap.put("key1", "value1"); osmTagMap.put("key2", "value2"); From f8db3ffd1a070d87dc343c61a87ed0b95ca27438 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Mon, 26 Mar 2018 14:32:13 -0700 Subject: [PATCH 28/39] Added a ProtoAtlas file suffix to FileSuffix per Mike suggestion --- .../atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java | 3 --- .../org/openstreetmap/atlas/streaming/resource/FileSuffix.java | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index fc962191f8..a74088c2bd 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -51,9 +51,6 @@ */ public class ProtoAtlasBuilder { - // File extension for naive protoatlas format - public static final String SUGGESTED_FILE_EXTENSION = ".npatlas"; - private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilder.class); /** diff --git a/src/main/java/org/openstreetmap/atlas/streaming/resource/FileSuffix.java b/src/main/java/org/openstreetmap/atlas/streaming/resource/FileSuffix.java index affa9855db..0e46f70044 100644 --- a/src/main/java/org/openstreetmap/atlas/streaming/resource/FileSuffix.java +++ b/src/main/java/org/openstreetmap/atlas/streaming/resource/FileSuffix.java @@ -21,6 +21,7 @@ public enum FileSuffix EXTENDED(".ext"), JSON(".json"), PBF(".pbf"), + PROTOATLAS(".patlas"), TEMPORARY(".tmp"), TEXT(".txt"), ZIP(".zip"), From 307b0158e575add8d1363c13f3e494ac5767d082 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Tue, 27 Mar 2018 13:43:35 -0700 Subject: [PATCH 29/39] Changed hardcoded protoc value in build.gradle --- build.gradle | 2 +- dependencies.gradle | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 8d39593fac..fc8d85b519 100644 --- a/build.gradle +++ b/build.gradle @@ -107,7 +107,7 @@ protobuf { generatedFilesBaseDir = 'gen/' protoc { - artifact = 'com.google.protobuf:protoc:2.6.1' + artifact = "${packages.protoc}" } generateProtoTasks { diff --git a/dependencies.gradle b/dependencies.gradle index 3cf606c0ce..caa4d6d9ae 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -21,6 +21,7 @@ project.ext.versions = [ jackson_core:'2.8.8', jackson_databind:'2.8.8.1', protobuf_java:'2.6.1', + protoc:'2.6.1' ] project.ext.packages = [ @@ -57,6 +58,7 @@ project.ext.packages = [ core: "com.fasterxml.jackson.core:jackson-core:${versions.jackson_core}", databind: "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" ], - protobuf_java: "com.google.protobuf:protobuf-java:2.6.1" + protobuf_java: "com.google.protobuf:protobuf-java:${versions.protobuf_java}", + protoc: "com.google.protobuf:protoc:${versions.protoc}" ] From 0e19a0b8c668f696553a7296b3421ece616584ea Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Wed, 28 Mar 2018 14:08:47 -0700 Subject: [PATCH 30/39] Tweak to buildflow. Gradle project refresh still requires a rebuild, but no longer forces user to manually update the build path --- build.gradle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fc8d85b519..8b1c76b151 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,7 @@ plugins apply from: 'dependencies.gradle' apply plugin: 'java' +apply plugin: 'eclipse' apply plugin: 'checkstyle' apply plugin: 'maven' apply plugin: 'signing' @@ -82,7 +83,7 @@ sourceSets { java { - srcDir file('gen/main/java') + srcDirs 'gen/main/java' } } } @@ -294,3 +295,10 @@ uploadArchives } } } + +eclipse.classpath.file { + whenMerged { classpath -> + def src = new org.gradle.plugins.ide.eclipse.model.SourceFolder('gen/main/java', null) + classpath.entries << src + } +} From 2678b776c289ee2b2cd473b0eeddefb88ef687bf Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Wed, 28 Mar 2018 15:58:37 -0700 Subject: [PATCH 31/39] Small tweak to prevent Gradle Refresh from deleting the generated code. --- build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8b1c76b151..30ba00bbb4 100644 --- a/build.gradle +++ b/build.gradle @@ -121,7 +121,9 @@ protobuf { } task deleteGeneratedProto { - delete generatedFilesBaseDir + // dummy task + // Since Eclipse's Gradle -> Refresh Gradle Project option runs all the clean commands + // we need to make sure this task doesn't delete the generated proto code } task recompileProto {} From bc6f4042642a317286182cf0334e0d4b7919d6c5 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 09:34:28 -0700 Subject: [PATCH 32/39] Added a task to actually clear the generated code. --- build.gradle | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 30ba00bbb4..76b699f781 100644 --- a/build.gradle +++ b/build.gradle @@ -120,10 +120,20 @@ protobuf { } } + task clearGeneratedProto { + delete generatedFilesBaseDir + } + task deleteGeneratedProto { - // dummy task - // Since Eclipse's Gradle -> Refresh Gradle Project option runs all the clean commands - // we need to make sure this task doesn't delete the generated proto code + // the Google gradle-protobuf plugin automatically registers this task + // to be run as a pre-requisite to 'clean'. Since Eclipse's "Refresh Gradle Project" + // directive runs the 'clean' task (which tries to delete the proto files), we must + // override this behaviour here + println "-------------------------------------------------" + println "Dummy task deleteGeneratedProto" + println "If you would like to actually clear the generated" + println "code, run task \'clearGeneratedProto\'" + println "-------------------------------------------------" } task recompileProto {} From fe1e9a1ecdd39f62a17268901e9c54217bdf1054 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 09:56:50 -0700 Subject: [PATCH 33/39] Slight tweak to build.gradle to fix beforeMerged --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 76b699f781..0ba420d26e 100644 --- a/build.gradle +++ b/build.gradle @@ -121,7 +121,7 @@ protobuf { } task clearGeneratedProto { - delete generatedFilesBaseDir + //delete generatedFilesBaseDir } task deleteGeneratedProto { @@ -309,7 +309,7 @@ uploadArchives } eclipse.classpath.file { - whenMerged { classpath -> + beforeMerged { classpath -> def src = new org.gradle.plugins.ide.eclipse.model.SourceFolder('gen/main/java', null) classpath.entries << src } From 3918bb8221bf4abf373054d4f8f100b510df40c7 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 10:22:34 -0700 Subject: [PATCH 34/39] Hopefully ready for merge --- build.gradle | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index 0ba420d26e..ea87269371 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,6 @@ plugins apply from: 'dependencies.gradle' apply plugin: 'java' -apply plugin: 'eclipse' apply plugin: 'checkstyle' apply plugin: 'maven' apply plugin: 'signing' @@ -121,7 +120,7 @@ protobuf { } task clearGeneratedProto { - //delete generatedFilesBaseDir + delete generatedFilesBaseDir } task deleteGeneratedProto { @@ -307,10 +306,3 @@ uploadArchives } } } - -eclipse.classpath.file { - beforeMerged { classpath -> - def src = new org.gradle.plugins.ide.eclipse.model.SourceFolder('gen/main/java', null) - classpath.entries << src - } -} From ee438011dc6ece547edd6550e001308a269cc430 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 13:28:36 -0700 Subject: [PATCH 35/39] Finalized fixes to gradle. Everything should work now. --- .gitignore | 2 +- ECLIPSE_README.md | 39 --------------------------------------- README.md | 7 +++++++ build.gradle | 23 +++++++---------------- 4 files changed, 15 insertions(+), 56 deletions(-) delete mode 100644 ECLIPSE_README.md diff --git a/.gitignore b/.gitignore index 1d0d551115..c48ee40aff 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,4 @@ **/out/ /bin/ **/ignored/ -gen/ +src/generated/ diff --git a/ECLIPSE_README.md b/ECLIPSE_README.md deleted file mode 100644 index 119a0ab785..0000000000 --- a/ECLIPSE_README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Working with Atlas in Eclipse - -## First time setup -1. Clone the Atlas repository from your fork into the desired local location. - -2. Import the project into Eclipse using "File" -> "Import" -> "Gradle" -> "Existing Gradle Project" - -3. From the command line, run -``` -$ cd /path/to/local/Atlas/repo -$ ./gradlew clean build -x test -x integrationTest -``` - -4. Back in Eclipse, right click the Atlas project in Package Explorer and click "Refresh" - 1. NOTE: You should now see a "gen" folder appear in the package explorer - -5. Again, right click on the Atlas project in Package Explorer and select "Build Path" -> "Configure Build Path" - -6. Select "Add Folder" then expand "gen" -> "main" and select the box next to "java" - -7. Click "Apply and close" - -8. The red lines and X's should all be resolved now. You're good to go! - -## How to rebuild the protobuf templates -1. You're here because you wanted to change the .proto template files, and now you need to regenerate and rebuild the code in "gen". Easy! - -2. Make the necessary adjustment to the .proto files first (in src/main/proto), but do not touch any of the dependant source code. Otherwise Eclipse will start complaining. - -3. Once you are done adjusting the .proto files, open a command line and run -``` -$ cd /path/to/local/Atlas/repo -$ ./gradlew clean build -x test -x integrationTest -``` - -4. Back in Eclipse, right click on the Atlas project in Package Explorer and click "Refresh" - -5. Now you can make changes to your actual source, which depends on the code generated from the .proto files. The Eclipse play button should work once again. - diff --git a/README.md b/README.md index 68348f1088..29b81592f8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,13 @@ Projects using Atlas: To contribute to the project, please see the [contributing guidelines](CONTRIBUTING.md). +After initially cloning the project, first perform +``` +$ cd /path/to/local/Atlas/repo +$ ./gradlew clean build -x check -x javadoc -x sources +``` +The '-x' options are not required but will make this initial build much faster. Then once the build is complete, right click the project in Eclipse and select 'Refresh'. Then, right click it again, scroll down to 'Gradle', and select 'Refresh Gradle Project'. The project should now be ready! + ## What's in it? * [`Edge`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Edge.java)s and [`Node`](src/main/java/org/openstreetmap/atlas/geography/atlas/items/Node.java)s for navigable items (Roads, Ferries) diff --git a/build.gradle b/build.gradle index ea87269371..e97632a8ca 100644 --- a/build.gradle +++ b/build.gradle @@ -82,7 +82,7 @@ sourceSets { java { - srcDirs 'gen/main/java' + srcDirs 'src/main/java', 'src/generated/main/java' } } } @@ -104,7 +104,7 @@ configurations protobuf { - generatedFilesBaseDir = 'gen/' + generatedFilesBaseDir = 'src/generated/' protoc { artifact = "${packages.protoc}" @@ -119,20 +119,11 @@ protobuf { } } - task clearGeneratedProto { - delete generatedFilesBaseDir - } - task deleteGeneratedProto { - // the Google gradle-protobuf plugin automatically registers this task - // to be run as a pre-requisite to 'clean'. Since Eclipse's "Refresh Gradle Project" - // directive runs the 'clean' task (which tries to delete the proto files), we must - // override this behaviour here - println "-------------------------------------------------" - println "Dummy task deleteGeneratedProto" - println "If you would like to actually clear the generated" - println "code, run task \'clearGeneratedProto\'" - println "-------------------------------------------------" + doLast{ + println "Deleting generated protos" + delete generatedFilesBaseDir + } } task recompileProto {} @@ -173,7 +164,7 @@ dependencies compile packages.jackson.core compile packages.jackson.databind compile packages.protobuf_java - + shaded project.configurations.getByName('compile') shaded packages.slf4j.log4j12 shaded packages.log4j From eeeb127cd68c30b335447b637fe20ac7e3dcfae4 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 13:42:01 -0700 Subject: [PATCH 36/39] Slight change to README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 29b81592f8..9f02f50dd7 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ To contribute to the project, please see the [contributing guidelines](CONTRIBUT After initially cloning the project, first perform ``` -$ cd /path/to/local/Atlas/repo +$ cd /path/to/local/atlas/repo $ ./gradlew clean build -x check -x javadoc -x sources ``` -The '-x' options are not required but will make this initial build much faster. Then once the build is complete, right click the project in Eclipse and select 'Refresh'. Then, right click it again, scroll down to 'Gradle', and select 'Refresh Gradle Project'. The project should now be ready! +The '-x' options are not required but will make this initial build much faster. Once the build is complete, you will need to import the project into Eclipse. Go to 'File' -> 'Import' -> 'Existing Gradle Project' and select 'Finish'. Now that your project shows up in Eclipse, right click the project and select 'Refresh'. Then, right click it again, scroll down to 'Gradle', and select 'Refresh Gradle Project'. The project should now be ready! ## What's in it? From 0d4e3f084c032f9890c8093e02e6f0b6d7cbabbc Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 13:52:45 -0700 Subject: [PATCH 37/39] Fixed checkstyle and spotless suppressions to deal with new generated directory. --- build.gradle | 2 +- config/checkstyle/suppressions.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index e97632a8ca..121b790997 100644 --- a/build.gradle +++ b/build.gradle @@ -43,7 +43,7 @@ spotless { java { target project.fileTree(project.rootDir) { include '**/*.java' - exclude 'gen/**/*.java' + exclude 'src/generated/**/*.java' } importOrder(['static java', 'static javax', 'static org', 'static com', 'static scala', 'java', 'javax', 'org', 'com', 'scala']) removeUnusedImports() diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index 5108491075..cc14be5ee3 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -10,5 +10,5 @@ - + From 9657baca759065a68e628a4883e477f9623e0d0c Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 14:54:53 -0700 Subject: [PATCH 38/39] Final changes before merge. Hopefully we are ready to go. --- .../builder/proto/ProtoAtlasBuilder.java | 83 +++++++------------ .../command/PackedToProtoAtlasSubCommand.java | 5 -- .../command/ProtoToPackedAtlasSubCommand.java | 5 -- .../proto/ProtoTagListConverter.java | 2 - src/main/proto/AtlasContainer.proto | 6 ++ .../proto/ProtoTagListConverterTest.java | 4 - 6 files changed, 36 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index a74088c2bd..3b5124a16c 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -24,6 +24,7 @@ import org.openstreetmap.atlas.geography.atlas.items.RelationMember; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlas; import org.openstreetmap.atlas.geography.atlas.packed.PackedAtlasBuilder; +import org.openstreetmap.atlas.geography.atlas.pbf.slicing.identifier.ReverseIdentifierFactory; import org.openstreetmap.atlas.geography.converters.proto.ProtoLocationConverter; import org.openstreetmap.atlas.geography.converters.proto.ProtoTagListConverter; import org.openstreetmap.atlas.proto.ProtoArea; @@ -38,8 +39,6 @@ import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.streaming.resource.WritableResource; import org.openstreetmap.atlas.utilities.collections.Maps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.protobuf.InvalidProtocolBufferException; @@ -51,7 +50,9 @@ */ public class ProtoAtlasBuilder { - private static final Logger logger = LoggerFactory.getLogger(ProtoAtlasBuilder.class); + private final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); + private final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); + private final ReverseIdentifierFactory reverseIDFactory = new ReverseIdentifierFactory(); /** * Read a resource in naive ProtoAtlas format into a PackedAtlas. @@ -125,77 +126,69 @@ public void write(final Atlas atlas, final WritableResource resource) private void parseAreas(final PackedAtlasBuilder builder, final List areas) { - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); areas.forEach(protoArea -> { final long identifier = protoArea.getId(); final List shapePoints = protoArea.getShapePointsList().stream() - .map(protoLocationConverter::convert).collect(Collectors.toList()); + .map(this.protoLocationConverter::convert).collect(Collectors.toList()); final Polygon geometry = new Polygon(shapePoints); - final Map tags = protoTagListConverter.convert(protoArea.getTagsList()); - // TODO see Mike's PR note about duplicate points, since start and end are the same for - // polygons. Make sure this is not happening. I think it's fine since I am building the - // Polygon using the Polygon constructor from a list of points that contains no - // duplicates. + final Map tags = this.protoTagListConverter + .convert(protoArea.getTagsList()); builder.addArea(identifier, geometry, tags); }); } private void parseEdges(final PackedAtlasBuilder builder, final List edges) { - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); edges.forEach(protoEdge -> { final long identifier = protoEdge.getId(); final List shapePoints = protoEdge.getShapePointsList().stream() - .map(protoLocationConverter::convert).collect(Collectors.toList()); + .map(this.protoLocationConverter::convert).collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); - final Map tags = protoTagListConverter.convert(protoEdge.getTagsList()); + final Map tags = this.protoTagListConverter + .convert(protoEdge.getTagsList()); builder.addEdge(identifier, geometry, tags); }); } private void parseLines(final PackedAtlasBuilder builder, final List lines) { - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); lines.forEach(protoLine -> { final long identifier = protoLine.getId(); final List shapePoints = protoLine.getShapePointsList().stream() - .map(protoLocationConverter::convert).collect(Collectors.toList()); + .map(this.protoLocationConverter::convert).collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); - final Map tags = protoTagListConverter.convert(protoLine.getTagsList()); + final Map tags = this.protoTagListConverter + .convert(protoLine.getTagsList()); builder.addLine(identifier, geometry, tags); }); } private void parseNodes(final PackedAtlasBuilder builder, final List nodes) { - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); nodes.forEach(protoNode -> { final long identifier = protoNode.getId(); final Longitude longitude = Longitude.dm7(protoNode.getLocation().getLongitude()); final Latitude latitude = Latitude.dm7(protoNode.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); - final Map tags = protoTagListConverter.convert(protoNode.getTagsList()); + final Map tags = this.protoTagListConverter + .convert(protoNode.getTagsList()); builder.addNode(identifier, geometry, tags); }); } private void parsePoints(final PackedAtlasBuilder builder, final List points) { - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); points.forEach(protoPoint -> { final long identifier = protoPoint.getId(); final Longitude longitude = Longitude.dm7(protoPoint.getLocation().getLongitude()); final Latitude latitude = Latitude.dm7(protoPoint.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); - final Map tags = protoTagListConverter + final Map tags = this.protoTagListConverter .convert(protoPoint.getTagsList()); builder.addPoint(identifier, geometry, tags); }); @@ -220,23 +213,20 @@ private RelationBean parseRelationBean(final ProtoRelation protoRelation) private void parseRelations(final PackedAtlasBuilder builder, final List relations) { - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); relations.forEach(protoRelation -> { final long identifier = protoRelation.getId(); final RelationBean bean = parseRelationBean(protoRelation); - final Map tags = protoTagListConverter + final Map tags = this.protoTagListConverter .convert(protoRelation.getTagsList()); - // TODO should identifier and OSMIdentifier be the same? - builder.addRelation(identifier, identifier, bean, tags); + builder.addRelation(identifier, this.reverseIDFactory.getOsmIdentifier(identifier), + bean, tags); }); } private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) { long numberOfAreas = 0; - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Area area : atlas.areas()) { @@ -244,11 +234,11 @@ private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuil protoAreaBuilder.setId(area.getIdentifier()); final List protoLocations = area.asPolygon().stream() - .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); + .map(this.protoLocationConverter::backwardConvert).collect(Collectors.toList()); protoAreaBuilder.addAllShapePoints(protoLocations); final Map tags = area.getTags(); - protoAreaBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoAreaBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); numberOfAreas++; protoAtlasBuilder.addAreas(protoAreaBuilder.build()); @@ -259,8 +249,6 @@ private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuil private void writeEdgesToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) { long numberOfEdges = 0; - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Edge edge : atlas.edges()) { @@ -268,11 +256,11 @@ private void writeEdgesToBuilder(final Atlas atlas, final Builder protoAtlasBuil protoEdgeBuilder.setId(edge.getIdentifier()); final List protoLocations = edge.asPolyLine().stream() - .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); + .map(this.protoLocationConverter::backwardConvert).collect(Collectors.toList()); protoEdgeBuilder.addAllShapePoints(protoLocations); final Map tags = edge.getTags(); - protoEdgeBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoEdgeBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); numberOfEdges++; protoAtlasBuilder.addEdges(protoEdgeBuilder.build()); @@ -284,8 +272,6 @@ private void writeLinesToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { long numberOfLines = 0; - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Line line : atlas.lines()) { @@ -293,11 +279,11 @@ private void writeLinesToBuilder(final Atlas atlas, protoLineBuilder.setId(line.getIdentifier()); final List protoLocations = line.asPolyLine().stream() - .map(protoLocationConverter::backwardConvert).collect(Collectors.toList()); + .map(this.protoLocationConverter::backwardConvert).collect(Collectors.toList()); protoLineBuilder.addAllShapePoints(protoLocations); final Map tags = line.getTags(); - protoLineBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoLineBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); numberOfLines++; protoAtlasBuilder.addLines(protoLineBuilder.build()); @@ -308,8 +294,6 @@ private void writeLinesToBuilder(final Atlas atlas, private void writeNodesToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) { long numberOfNodes = 0; - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Node node : atlas.nodes()) { @@ -317,10 +301,10 @@ private void writeNodesToBuilder(final Atlas atlas, final Builder protoAtlasBuil protoNodeBuilder.setId(node.getIdentifier()); protoNodeBuilder - .setLocation(protoLocationConverter.backwardConvert(node.getLocation())); + .setLocation(this.protoLocationConverter.backwardConvert(node.getLocation())); final Map tags = node.getTags(); - protoNodeBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoNodeBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); numberOfNodes++; protoAtlasBuilder.addNodes(protoNodeBuilder.build()); @@ -332,8 +316,6 @@ private void writePointsToBuilder(final Atlas atlas, final ProtoAtlasContainer.Builder protoAtlasBuilder) { long numberOfPoints = 0; - final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Point point : atlas.points()) { @@ -341,10 +323,10 @@ private void writePointsToBuilder(final Atlas atlas, protoPointBuilder.setId(point.getIdentifier()); protoPointBuilder - .setLocation(protoLocationConverter.backwardConvert(point.getLocation())); + .setLocation(this.protoLocationConverter.backwardConvert(point.getLocation())); final Map tags = point.getTags(); - protoPointBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoPointBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); numberOfPoints++; protoAtlasBuilder.addPoints(protoPointBuilder.build()); @@ -355,7 +337,6 @@ private void writePointsToBuilder(final Atlas atlas, private void writeRelationsToBuilder(final Atlas atlas, final Builder protoAtlasBuilder) { long numberOfRelations = 0; - final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); for (final Relation relation : atlas.relations()) { @@ -369,15 +350,11 @@ private void writeRelationsToBuilder(final Atlas atlas, final Builder protoAtlas beanBuilder.setMemberRole(member.getRole()); final ItemType type = ItemType.forEntity(member.getEntity()); beanBuilder.setMemberType(ProtoRelation.ProtoItemType.valueOf(type.getValue())); - - // TODO there may be a problem here if there are more relations in this atlas than - // possible int32 values. This is because protobuf internally stores the protobeans - // list as an ArrayList protoRelationBuilder.addBeans(beanBuilder.build()); } final Map tags = relation.getTags(); - protoRelationBuilder.addAllTags(protoTagListConverter.backwardConvert(tags)); + protoRelationBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); numberOfRelations++; protoAtlasBuilder.addRelations(protoRelationBuilder.build()); diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java index 4afa6fd8d3..5054efa323 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/PackedToProtoAtlasSubCommand.java @@ -14,8 +14,6 @@ import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Command for converting a serialized PackedAtlas to a serialized ProtoAtlas @@ -24,9 +22,6 @@ */ public class PackedToProtoAtlasSubCommand implements FlexibleSubCommand { - private static final Logger logger = LoggerFactory - .getLogger(PackedToProtoAtlasSubCommand.class); - private static final String NAME = "packed-to-proto"; private static final String DESCRIPTION = "converts a packed atlas to a naive proto-based atlas"; private static final String PACKED_SWITCH_TEXT = "packed-atlas"; diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java index 39e722dab2..e3577a5640 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/command/ProtoToPackedAtlasSubCommand.java @@ -14,8 +14,6 @@ import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList; import org.openstreetmap.atlas.utilities.runtime.CommandMap; import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Command for converting a serialized ProtoAtlas to a serialized PackedAtlas @@ -24,9 +22,6 @@ */ public class ProtoToPackedAtlasSubCommand implements FlexibleSubCommand { - private static final Logger logger = LoggerFactory - .getLogger(ProtoToPackedAtlasSubCommand.class); - private static final String NAME = "proto-to-packed"; private static final String DESCRIPTION = "converts a naive proto-based atlas to a packed atlas"; private static final String PROTO_SWITCH_TEXT = "proto-atlas"; diff --git a/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java index 8362c528ba..e30f7ba848 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java +++ b/src/main/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverter.java @@ -20,8 +20,6 @@ public class ProtoTagListConverter implements TwoWayConverter, Ma { private static final Logger logger = LoggerFactory.getLogger(ProtoTagListConverter.class); - private static final String TAG_DELIMITER = "="; - @Override public List backwardConvert(final Map osmTagMap) { diff --git a/src/main/proto/AtlasContainer.proto b/src/main/proto/AtlasContainer.proto index 23a6ff1fd2..d0fc8f00f8 100644 --- a/src/main/proto/AtlasContainer.proto +++ b/src/main/proto/AtlasContainer.proto @@ -12,6 +12,12 @@ import "Node.proto"; import "Edge.proto"; import "Relation.proto"; +/* + * NOTE: the 'repeated' directive is cpmpiled into Java as an ArrayList. + * The ArrayList implementation has a 32bit signed integer size limit. + * This could be a potential problem for atlases with an absurd number + * of a specific feature. + */ message ProtoAtlasContainer { optional int64 numberOfPoints = 1; repeated ProtoPoint points = 2; diff --git a/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java index 1925aa2420..b22ea6fbe4 100644 --- a/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java +++ b/src/test/java/org/openstreetmap/atlas/geography/converters/proto/ProtoTagListConverterTest.java @@ -8,16 +8,12 @@ import org.junit.Assert; import org.junit.Test; import org.openstreetmap.atlas.proto.ProtoTag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @author lcram */ public class ProtoTagListConverterTest { - private static final Logger logger = LoggerFactory.getLogger(ProtoTagListConverterTest.class); - @Test public void testEmptyConversion() { From 79e74503ae795b442f6b85270762425e86f7c5f8 Mon Sep 17 00:00:00 2001 From: lucaspcram Date: Thu, 29 Mar 2018 15:42:09 -0700 Subject: [PATCH 39/39] Fixed some nits. --- .../builder/proto/ProtoAtlasBuilder.java | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java index 3b5124a16c..e88f02a98b 100644 --- a/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java +++ b/src/main/java/org/openstreetmap/atlas/geography/atlas/builder/proto/ProtoAtlasBuilder.java @@ -50,9 +50,9 @@ */ public class ProtoAtlasBuilder { - private final ProtoLocationConverter protoLocationConverter = new ProtoLocationConverter(); - private final ProtoTagListConverter protoTagListConverter = new ProtoTagListConverter(); - private final ReverseIdentifierFactory reverseIDFactory = new ReverseIdentifierFactory(); + private static final ProtoLocationConverter PROTOLOCATION_CONVERTER = new ProtoLocationConverter(); + private static final ProtoTagListConverter PROTOTAG_LIST_CONVERTER = new ProtoTagListConverter(); + private static final ReverseIdentifierFactory REVERSE_IDENTIFIER_FACTORY = new ReverseIdentifierFactory(); /** * Read a resource in naive ProtoAtlas format into a PackedAtlas. @@ -130,9 +130,10 @@ private void parseAreas(final PackedAtlasBuilder builder, final List { final long identifier = protoArea.getId(); final List shapePoints = protoArea.getShapePointsList().stream() - .map(this.protoLocationConverter::convert).collect(Collectors.toList()); + .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::convert) + .collect(Collectors.toList()); final Polygon geometry = new Polygon(shapePoints); - final Map tags = this.protoTagListConverter + final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoArea.getTagsList()); builder.addArea(identifier, geometry, tags); }); @@ -144,9 +145,10 @@ private void parseEdges(final PackedAtlasBuilder builder, final List { final long identifier = protoEdge.getId(); final List shapePoints = protoEdge.getShapePointsList().stream() - .map(this.protoLocationConverter::convert).collect(Collectors.toList()); + .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::convert) + .collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); - final Map tags = this.protoTagListConverter + final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoEdge.getTagsList()); builder.addEdge(identifier, geometry, tags); }); @@ -158,9 +160,10 @@ private void parseLines(final PackedAtlasBuilder builder, final List { final long identifier = protoLine.getId(); final List shapePoints = protoLine.getShapePointsList().stream() - .map(this.protoLocationConverter::convert).collect(Collectors.toList()); + .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::convert) + .collect(Collectors.toList()); final PolyLine geometry = new PolyLine(shapePoints); - final Map tags = this.protoTagListConverter + final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoLine.getTagsList()); builder.addLine(identifier, geometry, tags); }); @@ -174,7 +177,7 @@ private void parseNodes(final PackedAtlasBuilder builder, final List final Longitude longitude = Longitude.dm7(protoNode.getLocation().getLongitude()); final Latitude latitude = Latitude.dm7(protoNode.getLocation().getLatitude()); final Location geometry = new Location(latitude, longitude); - final Map tags = this.protoTagListConverter + final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoNode.getTagsList()); builder.addNode(identifier, geometry, tags); }); @@ -188,7 +191,7 @@ private void parsePoints(final PackedAtlasBuilder builder, final List tags = this.protoTagListConverter + final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoPoint.getTagsList()); builder.addPoint(identifier, geometry, tags); }); @@ -217,10 +220,11 @@ private void parseRelations(final PackedAtlasBuilder builder, { final long identifier = protoRelation.getId(); final RelationBean bean = parseRelationBean(protoRelation); - final Map tags = this.protoTagListConverter + final Map tags = ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER .convert(protoRelation.getTagsList()); - builder.addRelation(identifier, this.reverseIDFactory.getOsmIdentifier(identifier), - bean, tags); + builder.addRelation(identifier, + ProtoAtlasBuilder.REVERSE_IDENTIFIER_FACTORY.getOsmIdentifier(identifier), bean, + tags); }); } @@ -234,11 +238,13 @@ private void writeAreasToBuilder(final Atlas atlas, final Builder protoAtlasBuil protoAreaBuilder.setId(area.getIdentifier()); final List protoLocations = area.asPolygon().stream() - .map(this.protoLocationConverter::backwardConvert).collect(Collectors.toList()); + .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::backwardConvert) + .collect(Collectors.toList()); protoAreaBuilder.addAllShapePoints(protoLocations); final Map tags = area.getTags(); - protoAreaBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); + protoAreaBuilder + .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfAreas++; protoAtlasBuilder.addAreas(protoAreaBuilder.build()); @@ -256,11 +262,13 @@ private void writeEdgesToBuilder(final Atlas atlas, final Builder protoAtlasBuil protoEdgeBuilder.setId(edge.getIdentifier()); final List protoLocations = edge.asPolyLine().stream() - .map(this.protoLocationConverter::backwardConvert).collect(Collectors.toList()); + .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::backwardConvert) + .collect(Collectors.toList()); protoEdgeBuilder.addAllShapePoints(protoLocations); final Map tags = edge.getTags(); - protoEdgeBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); + protoEdgeBuilder + .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfEdges++; protoAtlasBuilder.addEdges(protoEdgeBuilder.build()); @@ -279,11 +287,13 @@ private void writeLinesToBuilder(final Atlas atlas, protoLineBuilder.setId(line.getIdentifier()); final List protoLocations = line.asPolyLine().stream() - .map(this.protoLocationConverter::backwardConvert).collect(Collectors.toList()); + .map(ProtoAtlasBuilder.PROTOLOCATION_CONVERTER::backwardConvert) + .collect(Collectors.toList()); protoLineBuilder.addAllShapePoints(protoLocations); final Map tags = line.getTags(); - protoLineBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); + protoLineBuilder + .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfLines++; protoAtlasBuilder.addLines(protoLineBuilder.build()); @@ -300,11 +310,12 @@ private void writeNodesToBuilder(final Atlas atlas, final Builder protoAtlasBuil final ProtoNode.Builder protoNodeBuilder = ProtoNode.newBuilder(); protoNodeBuilder.setId(node.getIdentifier()); - protoNodeBuilder - .setLocation(this.protoLocationConverter.backwardConvert(node.getLocation())); + protoNodeBuilder.setLocation( + ProtoAtlasBuilder.PROTOLOCATION_CONVERTER.backwardConvert(node.getLocation())); final Map tags = node.getTags(); - protoNodeBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); + protoNodeBuilder + .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfNodes++; protoAtlasBuilder.addNodes(protoNodeBuilder.build()); @@ -322,11 +333,12 @@ private void writePointsToBuilder(final Atlas atlas, final ProtoPoint.Builder protoPointBuilder = ProtoPoint.newBuilder(); protoPointBuilder.setId(point.getIdentifier()); - protoPointBuilder - .setLocation(this.protoLocationConverter.backwardConvert(point.getLocation())); + protoPointBuilder.setLocation( + ProtoAtlasBuilder.PROTOLOCATION_CONVERTER.backwardConvert(point.getLocation())); final Map tags = point.getTags(); - protoPointBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); + protoPointBuilder + .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfPoints++; protoAtlasBuilder.addPoints(protoPointBuilder.build()); @@ -351,10 +363,10 @@ private void writeRelationsToBuilder(final Atlas atlas, final Builder protoAtlas final ItemType type = ItemType.forEntity(member.getEntity()); beanBuilder.setMemberType(ProtoRelation.ProtoItemType.valueOf(type.getValue())); protoRelationBuilder.addBeans(beanBuilder.build()); - } final Map tags = relation.getTags(); - protoRelationBuilder.addAllTags(this.protoTagListConverter.backwardConvert(tags)); + protoRelationBuilder + .addAllTags(ProtoAtlasBuilder.PROTOTAG_LIST_CONVERTER.backwardConvert(tags)); numberOfRelations++; protoAtlasBuilder.addRelations(protoRelationBuilder.build());