Feature concatenate geojson with object properties (#205)

* Update ShardFileOverlapsPolygon to match shard resources of other types * Handle null (empty) subatlas after filtering single resource in load() * Use org.openstreetmap.atlas.utilities.collections.Iterables * Add prototype changes for ConcatenateGeoJsonCommand * Concatenate geojson files while preserving data types of properties * Sort input files to make output order deterministic * Apply spotless java
osmlab · Aug 28, 2018 · 8ef8677 · 8ef8677
1 parent 66273bc
commit 8ef8677
Show file tree

Hide file tree

Showing 8 changed files with 203 additions and 42 deletions.
diff --git a/src/main/java/org/openstreetmap/atlas/geography/geojson/ConcatenateGeoJsonCommand.java b/src/main/java/org/openstreetmap/atlas/geography/geojson/ConcatenateGeoJsonCommand.java
@@ -1,14 +1,9 @@
 package org.openstreetmap.atlas.geography.geojson;
 
-import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
 import org.openstreetmap.atlas.exception.CoreException;
-import org.openstreetmap.atlas.geography.Location;
-import org.openstreetmap.atlas.geography.PolyLine;
-import org.openstreetmap.atlas.geography.geojson.GeoJsonBuilder.LocationIterableProperties;
 import org.openstreetmap.atlas.streaming.readers.GeoJsonReader;
 import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated;
 import org.openstreetmap.atlas.streaming.resource.File;
@@ -21,8 +16,6 @@
 import org.openstreetmap.atlas.utilities.runtime.Command;
 import org.openstreetmap.atlas.utilities.runtime.CommandMap;
 
-import com.google.gson.JsonElement;
-
 /**
  * Utility class to concatenate GeoJson objects.
  *
@@ -70,16 +63,19 @@ protected int onRun(final CommandMap command)
         final Mode mode = (Mode) command.get(MODE);
         final String filePrefix = (String) command.get(FILE_PREFIX);
 
-        final Iterable<LocationIterableProperties> result = readGeoJsonItems(mode,
-                folder.listFilesRecursively(), filePrefix);
-        final GeoJsonObject object = new GeoJsonBuilder().create(result);
+        // processing the files in sorted order makes testing easier
+        final List<File> files = folder.listFilesRecursively();
+        Collections.sort(files);
+        final Iterable<PropertiesLocated> jsonItems = readGeoJsonItems(mode, files, filePrefix);
+        final GeoJsonObject result = new GeoJsonBuilder()
+                .createFeatureCollectionFromPropertiesLocated(jsonItems);
         final JsonWriter writer = new JsonWriter(output);
-        writer.write(object.jsonObject());
+        writer.write(result.jsonObject());
         writer.close();
         return 0;
     }
 
-    protected Iterable<LocationIterableProperties> readGeoJsonItems(final Mode mode,
+    protected Iterable<PropertiesLocated> readGeoJsonItems(final Mode mode,
             final Iterable<File> files, final String filePrefix)
     {
         switch (mode)
@@ -103,37 +99,14 @@ protected SwitchList switches()
         return new SwitchList().with(PATH, OUTPUT, MODE, FILE_PREFIX);
     }
 
-    private Iterable<LocationIterableProperties> readGeoJsonItems(final Resource resource)
+    private Iterable<PropertiesLocated> readGeoJsonItems(final Resource resource)
     {
-        final Iterable<PropertiesLocated> read = () -> new GeoJsonReader(resource);
-        return Iterables.stream(read).flatMap(propertiesLocated ->
-        {
-            final Map<String, String> tags = new HashMap<>();
-            for (final Map.Entry<String, JsonElement> entry : propertiesLocated.getProperties()
-                    .entrySet())
-            {
-                final String key = entry.getKey();
-                final String value = entry.getValue().getAsString();
-                tags.put(key, value);
-            }
-            final List<LocationIterableProperties> result = new ArrayList<>();
-            if (propertiesLocated.getItem() instanceof Location)
-            {
-                final LocationIterableProperties item = new LocationIterableProperties(
-                        (Location) propertiesLocated.getItem(), tags);
-                result.add(item);
-            }
-            if (propertiesLocated.getItem() instanceof PolyLine)
-            {
-                final LocationIterableProperties item = new LocationIterableProperties(
-                        (PolyLine) propertiesLocated.getItem(), tags);
-                result.add(item);
-            }
-            return result;
-        });
+        final Iterable<PropertiesLocated> iterableOfPropertiesLocated = () -> new GeoJsonReader(
+                resource);
+        return iterableOfPropertiesLocated;
     }
 
-    private Iterable<LocationIterableProperties> readGeoJsonItems(final String line)
+    private Iterable<PropertiesLocated> readGeoJsonItems(final String line)
     {
         return readGeoJsonItems(new StringResource(line));
     }

diff --git a/src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonBuilder.java b/src/main/java/org/openstreetmap/atlas/geography/geojson/GeoJsonBuilder.java
@@ -7,9 +7,11 @@
 import java.util.Optional;
 
 import org.openstreetmap.atlas.exception.CoreException;
+import org.openstreetmap.atlas.geography.Located;
 import org.openstreetmap.atlas.geography.Location;
 import org.openstreetmap.atlas.geography.PolyLine;
 import org.openstreetmap.atlas.geography.Polygon;
+import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated;
 import org.openstreetmap.atlas.utilities.collections.Iterables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -374,6 +376,54 @@ public GeoJsonObject createFeatureCollection(final Iterable<GeoJsonObject> objec
         return new GeoJsonObject(result);
     }
 
+    /**
+     * Creates a GeoJson FeatureCollection containing a list of Features from an iterable of
+     * PropertiesLocated.
+     *
+     * @param iterableOfPropertiesLocated
+     *            iterable of PropertiesLocated
+     * @return a GeoJson FeatureCollection
+     */
+    public GeoJsonObject createFeatureCollectionFromPropertiesLocated(
+            final Iterable<PropertiesLocated> iterableOfPropertiesLocated)
+    {
+        final JsonObject result = new JsonObject();
+        result.addProperty(TYPE, FEATURE_COLLECTION);
+        final JsonArray features = new JsonArray();
+        int counter = 0;
+        for (final PropertiesLocated propertiesLocated : iterableOfPropertiesLocated)
+        {
+            if (this.logFrequency > 0 && ++counter % this.logFrequency == 0)
+            {
+                logger.info("Processed {} features.", counter);
+            }
+            final GeoJsonObject feature;
+            final Located located = propertiesLocated.getItem();
+            if (located instanceof Location)
+            {
+                feature = create((Location) located);
+            }
+            else if (located instanceof PolyLine)
+            {
+                feature = create((PolyLine) located);
+            }
+            else if (located instanceof Polygon)
+            {
+                feature = create((Polygon) located);
+            }
+            else
+            {
+                throw new CoreException("Unrecognized object type {}",
+                        located.getClass().getName());
+            }
+            final JsonObject featureJsonObj = feature.jsonObject();
+            featureJsonObj.add(PROPERTIES, propertiesLocated.getProperties());
+            features.add(feature.jsonObject());
+        }
+        result.add(FEATURES, features);
+        return new GeoJsonObject(result);
+    }
+
     /**
      * Creates a GeoJson FeatureCollection from an iterable of GeoJsonObject
      *

diff --git a/src/main/java/org/openstreetmap/atlas/streaming/resource/File.java b/src/main/java/org/openstreetmap/atlas/streaming/resource/File.java
@@ -26,7 +26,7 @@
  *
  * @author matthieun
  */
-public class File extends AbstractWritableResource
+public class File extends AbstractWritableResource implements Comparable<File>
 {
     private final java.io.File file;
     private String name = null;
@@ -129,6 +129,12 @@ public File child(final String name)
         return new File(getAbsolutePath() + "/" + name);
     }
 
+    @Override
+    public int compareTo(final File other)
+    {
+        return this.getFile().compareTo(other.getFile());
+    }
+
     public void delete()
     {
         try

diff --git a/src/test/java/org/openstreetmap/atlas/geography/geojson/ConcatenateGeoJsonCommandTest.java b/src/test/java/org/openstreetmap/atlas/geography/geojson/ConcatenateGeoJsonCommandTest.java
@@ -0,0 +1,65 @@
+package org.openstreetmap.atlas.geography.geojson;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Test cases for ConcatenateGeoJsonFiles.
+ * 
+ * @author rmegraw
+ */
+public class ConcatenateGeoJsonCommandTest
+{
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    @Test
+    public void testConcatenateGeoJsonFiles() throws IOException
+    {
+        final String inputPath = new File(this.getClass().getResource("test1.geojson").getPath())
+                .getParent();
+        final String outputPath = this.tempFolder.newFile().getAbsolutePath();
+
+        new ConcatenateGeoJsonCommand().runWithoutQuitting("-path=" + inputPath, "-mode=FILE",
+                "-output=" + outputPath);
+
+        final File outputFile = new File(outputPath);
+        Assert.assertTrue(outputFile.exists());
+
+        Assert.assertTrue(FileUtils
+                .readFileToString(new File(this.getClass()
+                        .getResource("concatenated_geojson_files_expected").getPath()),
+                        Charset.defaultCharset())
+                .equals(FileUtils.readFileToString(outputFile, Charset.defaultCharset())));
+
+    }
+
+    @Test
+    public void testConcatenateGeoJsonLines() throws IOException
+    {
+        final String inputPath = new File(this.getClass().getResource("test1.geojson").getPath())
+                .getParent();
+        final String outputPath = this.tempFolder.newFile().getAbsolutePath();
+
+        new ConcatenateGeoJsonCommand().runWithoutQuitting("-path=" + inputPath, "-mode=LINE",
+                "-output=" + outputPath, "-filePrefix=" + "test");
+
+        final File outputFile = new File(outputPath);
+        Assert.assertTrue(outputFile.exists());
+
+        Assert.assertTrue(FileUtils
+                .readFileToString(new File(this.getClass()
+                        .getResource("concatenated_geojson_files_expected").getPath()),
+                        Charset.defaultCharset())
+                .equals(FileUtils.readFileToString(outputFile, Charset.defaultCharset())));
+
+    }
+
+}
diff --git a/src/test/java/org/openstreetmap/atlas/geography/geojson/GeoJsonBuilderTest.java b/src/test/java/org/openstreetmap/atlas/geography/geojson/GeoJsonBuilderTest.java
@@ -18,7 +18,9 @@
 import org.openstreetmap.atlas.streaming.readers.GeoJsonReader;
 import org.openstreetmap.atlas.streaming.readers.json.serializers.PropertiesLocated;
 import org.openstreetmap.atlas.streaming.resource.StringResource;
+import org.openstreetmap.atlas.utilities.collections.Iterables;
 
+import com.google.gson.Gson;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 
@@ -46,6 +48,68 @@ public void testConsistency()
         Assert.assertEquals(polygon, item.getItem());
     }
 
+    @Test
+    public void testCreateFeatureCollectionFromPropertiesLocated()
+    {
+        final Map<String, Object> properties = new HashMap<>();
+        properties.put("prop1", "foo");
+        properties.put("prop2", new Float[] { 1.0F, 2.0F, 3.0F });
+        properties.put("prop3", 0);
+
+        final JsonObject propertiesObject = new JsonObject();
+        final Gson gson = new Gson();
+        properties.forEach((key, value) -> propertiesObject.add(key, gson.toJsonTree(value)));
+        propertiesObject.add("properties", propertiesObject);
+
+        final PropertiesLocated propertiesLocated1 = new PropertiesLocated(PolyLine.TEST_POLYLINE,
+                propertiesObject);
+        final PropertiesLocated propertiesLocated2 = new PropertiesLocated(PolyLine.TEST_POLYLINE,
+                propertiesObject);
+
+        final GeoJsonObject featureCollection = new GeoJsonBuilder()
+                .createFeatureCollectionFromPropertiesLocated(
+                        Iterables.from(propertiesLocated1, propertiesLocated2));
+
+        Assert.assertEquals("FeatureCollection",
+                featureCollection.jsonObject().get("type").getAsString());
+        Assert.assertEquals(2,
+                featureCollection.jsonObject().get("features").getAsJsonArray().size());
+        for (int i = 0; i < 2; i++)
+        {
+            Assert.assertEquals("Feature", featureCollection.jsonObject().get("features")
+                    .getAsJsonArray().get(i).getAsJsonObject().get("type").getAsString());
+            Assert.assertEquals("LineString",
+                    featureCollection.jsonObject().get("features").getAsJsonArray().get(i)
+                            .getAsJsonObject().get("geometry").getAsJsonObject().get("type")
+                            .getAsString());
+            Assert.assertEquals(
+                    PolyLine.TEST_POLYLINE.asGeoJson().jsonObject().get("features").getAsJsonArray()
+                            .get(0).getAsJsonObject().get("geometry"),
+                    featureCollection.jsonObject().get("features").getAsJsonArray().get(i)
+                            .getAsJsonObject().get("geometry").getAsJsonObject());
+            Assert.assertEquals(properties.get("prop1"),
+                    featureCollection.jsonObject().get("features").getAsJsonArray().get(i)
+                            .getAsJsonObject().get("properties").getAsJsonObject().get("prop1")
+                            .getAsString());
+            Assert.assertEquals(((Float[]) properties.get("prop2")).length,
+                    featureCollection.jsonObject().get("features").getAsJsonArray().get(i)
+                            .getAsJsonObject().get("properties").getAsJsonObject().get("prop2")
+                            .getAsJsonArray().size());
+            for (int j = i; j < 3; j++)
+            {
+                Assert.assertEquals(((Float[]) properties.get("prop2"))[j],
+                        featureCollection.jsonObject().get("features").getAsJsonArray().get(i)
+                                .getAsJsonObject().get("properties").getAsJsonObject().get("prop2")
+                                .getAsJsonArray().get(j).getAsFloat(),
+                        0D);
+            }
+            Assert.assertEquals(properties.get("prop3"),
+                    featureCollection.jsonObject().get("features").getAsJsonArray().get(i)
+                            .getAsJsonObject().get("properties").getAsJsonObject().get("prop3")
+                            .getAsInt());
+        }
+    }
+
     @Test
     public void testCreateFromGeometries()
     {

diff --git a/...t/resources/org/openstreetmap/atlas/geography/geojson/concatenated_geojson_files_expected b/...t/resources/org/openstreetmap/atlas/geography/geojson/concatenated_geojson_files_expected
@@ -0,0 +1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"LineString","coordinates":[[-71.0,-37.0],[-72.0,-38.0]]},"properties":{"prop1":"foo","prop2":1.99,"prop3":[1,2,3]}},{"type":"Feature","geometry":{"type":"LineString","coordinates":[[-72.0,-38.0],[-73.0,-39.0]]},"properties":{"prop1":"bar","prop2":2.99,"prop3":[2,3,4]}}]}
diff --git a/src/test/resources/org/openstreetmap/atlas/geography/geojson/test1.geojson b/src/test/resources/org/openstreetmap/atlas/geography/geojson/test1.geojson
@@ -0,0 +1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"LineString","coordinates":[[-71,-37],[-72,-38]]},"properties":{"prop1":"foo","prop2":1.99,"prop3":[1,2,3]}}]}
diff --git a/src/test/resources/org/openstreetmap/atlas/geography/geojson/test2.geojson b/src/test/resources/org/openstreetmap/atlas/geography/geojson/test2.geojson
@@ -0,0 +1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"LineString","coordinates":[[-72,-38],[-73,-39]]},"properties":{"prop1":"bar","prop2":2.99,"prop3":[2,3,4]}}]}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"LineString","coordinates":[[-71.0,-37.0],[-72.0,-38.0]]},"properties":{"prop1":"foo","prop2":1.99,"prop3":[1,2,3]}},{"type":"Feature","geometry":{"type":"LineString","coordinates":[[-72.0,-38.0],[-73.0,-39.0]]},"properties":{"prop1":"bar","prop2":2.99,"prop3":[2,3,4]}}]}