Skip to content

Commit

Permalink
PBF To Atlas Sub Commmand (#223)
Browse files Browse the repository at this point in the history 
* add PBF to Atlas command

* remove bounds param

* remove area param

* resources

* remove paramaters; add unit tests

* java docs

* change test name

* refactor .get to variable

* refactor class name

* import command.switch

* use this

* clarify switches

* remove shp file and test

* change to raw workflow
  • Loading branch information
@Bentleysb @matthieun
Bentleysb authored and matthieun committed Sep 25, 2018
1 parent e2508ab commit 36f9816
Show file tree
Hide file tree
Showing 9 changed files with 422 additions and 0 deletions.
210 changes: 210 additions & 0 deletions src/main/java/org/openstreetmap/atlas/geography/atlas/command/OsmPbfToAtlasSubCommand.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
package org.openstreetmap.atlas.geography.atlas.command;

import java.io.PrintStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.commons.io.FilenameUtils;
import org.openstreetmap.atlas.geography.MultiPolygon;
import org.openstreetmap.atlas.geography.atlas.Atlas;
import org.openstreetmap.atlas.geography.atlas.pbf.AtlasLoadingOption;
import org.openstreetmap.atlas.geography.atlas.raw.creation.RawAtlasGenerator;
import org.openstreetmap.atlas.geography.atlas.raw.sectioning.WaySectionProcessor;
import org.openstreetmap.atlas.geography.atlas.raw.slicing.RawAtlasCountrySlicer;
import org.openstreetmap.atlas.geography.boundary.CountryBoundaryMap;
import org.openstreetmap.atlas.streaming.resource.File;
import org.openstreetmap.atlas.tags.filters.ConfiguredTaggableFilter;
import org.openstreetmap.atlas.utilities.configuration.StandardConfiguration;
import org.openstreetmap.atlas.utilities.runtime.Command.Optionality;
import org.openstreetmap.atlas.utilities.runtime.Command.Switch;
import org.openstreetmap.atlas.utilities.runtime.Command.SwitchList;
import org.openstreetmap.atlas.utilities.runtime.CommandMap;
import org.openstreetmap.atlas.utilities.runtime.FlexibleSubCommand;

/**
* This command converts an OSM PBF file to an Atlas file. It requires the path to a pbf and an
* output file. It also takes a number of optional parameters.
*
* @author bbreithaupt
*/
public class OsmPbfToAtlasSubCommand implements FlexibleSubCommand
{
private static final String NAME = "pbf-to-atlas";
private static final String DESCRIPTION = "Converts a PBF to an Atlas file.";

// Required parameters
private static final Switch<File> INPUT_PARAMETER = new Switch<>("pbf", "Input PBF path",
File::new, Optionality.REQUIRED);
private static final Switch<File> OUTPUT_PARAMETER = new Switch<>("output",
"Output Atlas file path", File::new, Optionality.REQUIRED);

// Filter parameters
private static final Switch<File> EDGE_FILTER_PARAMETER = new Switch<>("edge-filter",
"Path to a local json filter for determining Edges", File::new, Optionality.OPTIONAL);
private static final Switch<File> NODE_FILTER_PARAMETER = new Switch<>("node-filter",
"Path to a local json filter for OSM nodes", File::new, Optionality.OPTIONAL);
private static final Switch<File> RELATION_FILTER_PARAMETER = new Switch<>("relation-filter",
"Path to a local json filter for OSM relations", File::new, Optionality.OPTIONAL);
private static final Switch<File> WAY_FILTER_PARAMETER = new Switch<>("way-filter",
"Path to a local json filter for OSM ways", File::new, Optionality.OPTIONAL);
private static final Switch<File> WAY_SECTION_FILTER_PARAMETER = new Switch<>(
"way-section-filter",
"Path to a local json filter for determining where to way section", File::new,
Optionality.OPTIONAL);

// Load Parameters
private static final Switch<Boolean> LOAD_RELATIONS_PARAMETER = new Switch<>("load-relations",
"Whether to load Relations (boolean)", Boolean::parseBoolean, Optionality.OPTIONAL,
"true");
private static final Switch<Boolean> LOAD_WAYS_PARAMETER = new Switch<>("load-ways",
"Whether to load ways (boolean)", Boolean::parseBoolean, Optionality.OPTIONAL, "true");

// Country parameters
private static final Switch<Set<String>> COUNTRY_CODES_PARAMETER = new Switch<>("country-codes",
"Countries from the country map to convert (comma separated ISO3 codes)",
code -> Arrays.stream(code.split(",")).collect(Collectors.toSet()),
Optionality.OPTIONAL);
private static final Switch<File> COUNTRY_MAP_PARAMETER = new Switch<>("country-boundary-map",
"Path to a local WKT or shp file containing a country boundary map", File::new,
Optionality.OPTIONAL);
private static final Switch<Boolean> COUNTRY_SLICING_PARAMETER = new Switch<>("country-slicing",
"Whether to perform country slicing (boolean)", Boolean::parseBoolean,
Optionality.OPTIONAL, "true");

@Override
public String getDescription()
{
return DESCRIPTION;
}

@Override
public String getName()
{
return NAME;
}

@Override
public SwitchList switches()
{
return new SwitchList().with(INPUT_PARAMETER, OUTPUT_PARAMETER, EDGE_FILTER_PARAMETER,
NODE_FILTER_PARAMETER, RELATION_FILTER_PARAMETER, WAY_FILTER_PARAMETER,
WAY_SECTION_FILTER_PARAMETER, LOAD_RELATIONS_PARAMETER, LOAD_WAYS_PARAMETER,
COUNTRY_CODES_PARAMETER, COUNTRY_MAP_PARAMETER, COUNTRY_SLICING_PARAMETER);
}

@Override
public void usage(final PrintStream writer)
{
writer.println("-pbf=/path/to/pbf : pbf to convert");
writer.println("-output=/path/to/output/atlas : Atlas file to output to");
writer.println("-edge-filter=/path/to/json/edge/filter : json filter to determine Edges");
writer.println("-node-filter=/path/to/json/node/filter : json filter for OSM nodes");
writer.println(
"-relation-filter=/path/to/json/relation/filter : json filter for OSM relations");
writer.println("-way-filter=/path/to/json/way/filter : json filter for OSM ways");
writer.println("-load-relations=boolean : whether to load Relations; defaults to true");
writer.println("-load-ways=boolean : whether to load ways; defaults to true");
writer.println(
"-country-codes=list,of,ISO3,codes : countries from the country map to convert");
writer.println(
"-country-boundary-map=/path/to/WKT/or/shp : a WKT or shp file containing a country boundary map");
writer.println(
"-country-slicing=boolean : whether to perform country slicing; defaults to true");
writer.println(
"-way-section-filter=/path/to/json/way/section/filter : json filter to determine where to way section");
}

@Override
public int execute(final CommandMap map)
{
final AtlasLoadingOption options = this.getAtlasLoadingOption(map);
Atlas atlas = new RawAtlasGenerator((File) map.get(INPUT_PARAMETER), options,
MultiPolygon.MAXIMUM).build();
if (options.isCountrySlicing())
{
atlas = new RawAtlasCountrySlicer(options.getCountryCodes(),
options.getCountryBoundaryMap()).slice(atlas);
}
atlas = new WaySectionProcessor(atlas, options).run();
atlas.save((File) map.get(OUTPUT_PARAMETER));
return 0;
}

/**
* Get or create a {@link CountryBoundaryMap}. If the country-boundary-map parameter is set,
* this will attempt to load the text or shape file from that parameter. Else, this will load
* using the entire world as the country UNK (unknown).
*
* @param map
* {@link CommandMap} containing the {@code COUNTRY_MAP_PARAMETER}
* @return {@link CountryBoundaryMap} loaded from a file or default
*/
private CountryBoundaryMap getCountryBoundaryMap(final CommandMap map)
{
final Optional<File> countryMapOption = (Optional<File>) map
.getOption(COUNTRY_MAP_PARAMETER);
CountryBoundaryMap countryMap = CountryBoundaryMap
.fromBoundaryMap(Collections.singletonMap("UNK", MultiPolygon.MAXIMUM));
if (countryMapOption.isPresent())
{
final File countryMapFile = countryMapOption.get();
if (FilenameUtils.isExtension(countryMapFile.getName(), "txt"))
{
countryMap = CountryBoundaryMap.fromPlainText(countryMapFile);
}
else if (FilenameUtils.isExtension(countryMapFile.getName(), "shp"))
{
countryMap = CountryBoundaryMap.fromShapeFile(countryMapFile.getFile());
}
}
return countryMap;
}

/**
* Creates an {@link AtlasLoadingOption} using configurable parameters. If any of the parameters
* are not set the defaults from {@link AtlasLoadingOption} are used.
*
* @param map
* {@link CommandMap}
* @return {@link AtlasLoadingOption}
*/
private AtlasLoadingOption getAtlasLoadingOption(final CommandMap map)
{
final CountryBoundaryMap countryMap = this.getCountryBoundaryMap(map);
final AtlasLoadingOption options = AtlasLoadingOption
.createOptionWithAllEnabled(countryMap);

// Set filters
map.getOption(EDGE_FILTER_PARAMETER).ifPresent(filter -> options.setEdgeFilter(
new ConfiguredTaggableFilter(new StandardConfiguration((File) filter))));
map.getOption(NODE_FILTER_PARAMETER).ifPresent(filter -> options.setOsmPbfNodeFilter(
new ConfiguredTaggableFilter(new StandardConfiguration((File) filter))));
map.getOption(RELATION_FILTER_PARAMETER)
.ifPresent(filter -> options.setOsmPbfRelationFilter(
new ConfiguredTaggableFilter(new StandardConfiguration((File) filter))));
map.getOption(WAY_FILTER_PARAMETER).ifPresent(filter -> options.setOsmPbfWayFilter(
new ConfiguredTaggableFilter(new StandardConfiguration((File) filter))));
map.getOption(WAY_SECTION_FILTER_PARAMETER).ifPresent(filter -> options.setWaySectionFilter(
new ConfiguredTaggableFilter(new StandardConfiguration((File) filter))));

// Set loading options
((Optional<Boolean>) map.getOption(LOAD_RELATIONS_PARAMETER))
.ifPresent(options::setLoadAtlasRelation);
((Optional<Boolean>) map.getOption(LOAD_WAYS_PARAMETER)).ifPresent(bool ->
{
options.setLoadAtlasLine(bool);
options.setLoadAtlasEdge(bool);
});

// Set country options
((Optional<Set>) map.getOption(COUNTRY_CODES_PARAMETER))
.ifPresent(options::setAdditionalCountryCodes);
((Optional<Boolean>) map.getOption(COUNTRY_SLICING_PARAMETER))
.ifPresent(options::setCountrySlicing);

return options;
}
}
176 changes: 176 additions & 0 deletions ...est/java/org/openstreetmap/atlas/geography/atlas/command/OsmPbfToAtlasSubCommandTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
package org.openstreetmap.atlas.geography.atlas.command;

import java.util.Collections;

import org.junit.Assert;
import org.junit.Test;
import org.openstreetmap.atlas.geography.atlas.Atlas;
import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader;
import org.openstreetmap.atlas.streaming.resource.File;

/**
* Unit tests for {@link OsmPbfToAtlasSubCommand}.
*
* @author bbreithaupt
*/
public class OsmPbfToAtlasSubCommandTest
{
private static String PBF = OsmPbfToAtlasSubCommandTest.class
.getResource("world_islands.osm.pbf").getPath();
private static String COUNTRY_BOUNDARY_MAP_TEXT = OsmPbfToAtlasSubCommandTest.class
.getResource("continent_map.txt").getPath();
private static String EDGE_FILTER = OsmPbfToAtlasSubCommandTest.class
.getResource("atlas-edge.json").getPath();
private static String WAY_SECTIONING_FILTER = OsmPbfToAtlasSubCommandTest.class
.getResource("atlas-way-section.json").getPath();
private static String NODE_FILTER = OsmPbfToAtlasSubCommandTest.class
.getResource("osm-pbf-node.json").getPath();
private static String RELATION_FILTER = OsmPbfToAtlasSubCommandTest.class
.getResource("osm-pbf-relation.json").getPath();
private static String WAY_FILTER = OsmPbfToAtlasSubCommandTest.class
.getResource("osm-pbf-way.json").getPath();

private static String ATLAS_NAME = "test_temp.atlas";

@Test
public void testDefaultConversion()
{
final File temp = File.temporaryFolder();

try
{
// Run OsmPbfToAtlasSubCommand
final String[] args = { "pbf-to-atlas", String.format("-pbf=%s", PBF),
String.format("-output=%s/%s", temp, ATLAS_NAME) };
new AtlasReader(args).runWithoutQuitting(args);

// Load new atlas
final Atlas atlas = new AtlasResourceLoader()
.load(new File(String.format("%s/%s", temp, ATLAS_NAME)));

// Test for way sectioning
Assert.assertNotNull(atlas.edge(87185620000002L));
// Test for country map
Assert.assertTrue(atlas.edge(87185039000000L).containsValue("iso_country_code",
Collections.singleton("UNK")));
// Inverse test for country codes
Assert.assertNotNull(atlas.point(1013787604000000L));
// Test default edge filter
Assert.assertNotNull(atlas.edge(87186304000001L));
// Test default filter
Assert.assertNotNull(atlas.point(3698322053000000L));
// Test default relation filter
Assert.assertNotNull(atlas.relation(2693943000000L));
// Test default way filter
Assert.assertNull(atlas.area(167578604000000L));
// Test default way section filter
Assert.assertNull(atlas.edge(87186195000018L));
}
finally
{
temp.deleteRecursively();
}
}

@Test
public void testFiltersTextMapCountryCodesConversion()
{
final File temp = File.temporaryFolder();

try
{
// Run OsmPbfToAtlasSubCommand
final String[] args = { "pbf-to-atlas", String.format("-pbf=%s", PBF),
String.format("-output=%s/%s", temp, ATLAS_NAME),
String.format("-country-boundary-map=%s", COUNTRY_BOUNDARY_MAP_TEXT),
"-country-codes=NAM,EUR", String.format("-edge-filter=%s", EDGE_FILTER),
String.format("-node-filter=%s", NODE_FILTER),
String.format("-relation-filter=%s", RELATION_FILTER),
String.format("-way-filter=%s", WAY_FILTER),
String.format("-way-section-filter=%s", WAY_SECTIONING_FILTER), };
new AtlasReader(args).runWithoutQuitting(args);

// Load new atlas
final Atlas atlas = new AtlasResourceLoader()
.load(new File(String.format("%s/%s", temp, ATLAS_NAME)));

// Test for way sectioning
Assert.assertNotNull(atlas.edge(87185620000002L));
// Test for country map
Assert.assertTrue(atlas.edge(87185039000000L).containsValue("iso_country_code",
Collections.singleton("NAM")));
// Test for country codes
Assert.assertNull(atlas.point(1013787604000000L));
// Test edge filter
Assert.assertNull(atlas.edge(87186304000001L));
// Test node filter
Assert.assertNull(atlas.point(3698322053000000L));
// Test relation filter
Assert.assertNull(atlas.relation(2693943000000L));
// Test way filter
Assert.assertNotNull(atlas.area(167578604000000L));
// Test way section filter
Assert.assertNotNull(atlas.edge(87186195000018L));
}
finally
{
temp.deleteRecursively();
}
}

@Test
public void testNoSlicingNoRelationsConversion()
{
final File temp = File.temporaryFolder();

try
{
// Run OsmPbfToAtlasSubCommand
final String[] args = { "pbf-to-atlas", String.format("-pbf=%s", PBF),
String.format("-output=%s/%s", temp, ATLAS_NAME),
String.format("-country-boundary-map=%s", COUNTRY_BOUNDARY_MAP_TEXT),
"-country-codes=NAM,EUR", "-country-slicing=false", "-load-relations=false" };
new AtlasReader(args).runWithoutQuitting(args);

// Load new atlas
final Atlas atlas = new AtlasResourceLoader()
.load(new File(String.format("%s/%s", temp, ATLAS_NAME)));
// Test for country slicing
Assert.assertFalse(atlas.edge(87185039000000L)
.containsKey(Collections.singleton("iso_country_code")));
// Test no load relation
Assert.assertFalse(atlas.relations().iterator().hasNext());
}
finally
{
temp.deleteRecursively();
}
}

@Test
public void testNoWaysConversion()
{
final File temp = File.temporaryFolder();

try
{
// Run OsmPbfToAtlasSubCommand
final String[] args = { "pbf-to-atlas", String.format("-pbf=%s", PBF),
String.format("-output=%s/%s", temp, ATLAS_NAME), "-load-ways=false" };
new AtlasReader(args).runWithoutQuitting(args);

// Load new atlas
final Atlas atlas = new AtlasResourceLoader()
.load(new File(String.format("%s/%s", temp, ATLAS_NAME)));

// Test no load ways
Assert.assertFalse(atlas.areas().iterator().hasNext());
Assert.assertFalse(atlas.edges().iterator().hasNext());
Assert.assertFalse(atlas.lines().iterator().hasNext());
}
finally
{
temp.deleteRecursively();
}
}
}
7 changes: 7 additions & 0 deletions src/test/resources/org/openstreetmap/atlas/geography/atlas/command/atlas-edge.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"filters": [
"access->!no|motor_vehicle->yes|motorcar->yes|vehicle->yes",
"oneway->!reversible",
"route->ferry|junction->roundabout|highway->MOTORWAY,TRUNK,PRIMARY,SECONDARY,TERTIARY,UNCLASSIFIED,RESIDENTIAL,SERVICE,MOTORWAY_LINK,TRUNK_LINK,PRIMARY_LINK,SECONDARY_LINK,TERTIARY_LINK,LIVING_STREET,PEDESTRIAN,TRACK,BUS_GUIDEWAY,RACEWAY,ROAD,FOOTWAY,BRIDLEWAY,STEPS,PATH,CYCLEWAY,ESCAPE"
]
}
Loading

0 comments on commit 36f9816

Please sign in to comment.