Skip to content

Commit

Permalink
Added CountryToShardList Caching (#202)
Browse files Browse the repository at this point in the history 
* Added CountryToShardListCache + test

* general cleanup

* cleanup

* Streamlined to one class, added more tests

* Removed unused country shard lists from unit test resource

* Made CountryToShardListCache a Command

* Added javadoc comments on public facing methods

* changed how exceptions are handled
  • Loading branch information
@matthieun
jwpgage authored and matthieun committed Aug 24, 2018
1 parent 65f5542 commit 85caca6
Show file tree
Hide file tree
Showing 3 changed files with 238 additions and 0 deletions.
173 changes: 173 additions & 0 deletions src/main/java/org/openstreetmap/atlas/geography/boundary/CountryToShardListCache.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
package org.openstreetmap.atlas.geography.boundary;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.openstreetmap.atlas.exception.CoreException;
import org.openstreetmap.atlas.geography.sharding.Sharding;
import org.openstreetmap.atlas.geography.sharding.SlippyTile;
import org.openstreetmap.atlas.streaming.resource.File;
import org.openstreetmap.atlas.streaming.resource.Resource;
import org.openstreetmap.atlas.streaming.resource.WritableResource;
import org.openstreetmap.atlas.streaming.writers.SafeBufferedWriter;
import org.openstreetmap.atlas.utilities.collections.StringList;
import org.openstreetmap.atlas.utilities.maps.MultiMap;
import org.openstreetmap.atlas.utilities.runtime.Command;
import org.openstreetmap.atlas.utilities.runtime.CommandMap;
import org.openstreetmap.atlas.utilities.time.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Class that stores country and intersecting shards in a {@link MultiMap}, and can be constructed
* from scratch or from file. Allows saving this mapping to file, which is useful when sharding is
* dense or borders are complex.
*
* @author james-gage
*/
public final class CountryToShardListCache extends Command
{
private static final Logger logger = LoggerFactory.getLogger(CountryToShardListCache.class);
private static final Switch<CountryBoundaryMap> BOUNDARIES = new Switch<>("boundaries",
"The country boundaries.", value -> initializeCountryBoundaryMap(value),
Optionality.REQUIRED);
private static final Switch<StringList> COUNTRIES = new Switch<>("countries",
"CSV list of the country iso3 codes", value -> StringList.split(value, ","),
Optionality.OPTIONAL);
private static final Switch<Sharding> SHARDING = new Switch<>("sharding",
"File containing the sharding definition (Works with files only)", Sharding::forString,
Optionality.REQUIRED);
private static final Switch<File> OUTPUT = new Switch<>("output", "The output file", File::new,
Optionality.REQUIRED);
private static final String DELIMITER = "||";
private final MultiMap<String, SlippyTile> countryToShards = new MultiMap<>();

public static void main(final String[] args)
{
new CountryToShardListCache().run(args);
}

private static CountryBoundaryMap initializeCountryBoundaryMap(final String value)
{
final Time start = Time.now();
logger.info("Loading boundaries");
final CountryBoundaryMap result = CountryBoundaryMap.fromPlainText(new File(value));
logger.info("Loaded boundaries in {}", start.elapsedSince());
return result;
}

public CountryToShardListCache(final CountryBoundaryMap boundaries, final Sharding sharding)
{
this(boundaries, new StringList(boundaries.allCountryNames()), sharding);
}

public CountryToShardListCache(final CountryBoundaryMap boundaries, final StringList countries,
final Sharding sharding)
{
CountryShardListing.countryToShardList(countries, boundaries, sharding)
.forEach((country, shardSet) ->
{
shardSet.forEach(shard -> this.countryToShards.add(country,
SlippyTile.forName(shard.getName())));
});
}

public CountryToShardListCache(final Resource resource)
{
try
{
resource.lines().forEach(line ->
{
final String[] countryAndShardList = line.split(Pattern.quote(DELIMITER));
final String country = countryAndShardList[0];
final String shardList = countryAndShardList[1];
Arrays.asList(shardList.split("\\s*,\\s*")).stream().map(SlippyTile::forName)
.forEach(slippyTile -> this.countryToShards.add(country, slippyTile));
});
}
catch (final Exception e)
{
throw new CoreException("Error while reading CountryToShardListCache resource", e);
}
}

private CountryToShardListCache()
{

}

/**
* Takes a country code, and returns a List of all {@link SlippyTile}s that cover the country
* boundary. If an invalid country code is passed, an empty list is returned.
*
* @param country
* The three digit country code
* @return A List of {@link SlippyTile}s
*/
public List<SlippyTile> getShardsForCountry(final String country)
{
if (this.countryToShards.containsKey(country))
{
return this.countryToShards.get(country);
}
else
{
return Collections.<SlippyTile> emptyList();
}
}

/**
* Writes to a {@link WritableResource} the CountryShardListCache. The resulting resource can be
* used to initialize another CountryShardListCache.
*
* @param output
* The {@link WritableResource} where the CountryToShardListCache will be written.
*/
public void save(final WritableResource output)
{
try (SafeBufferedWriter writer = output.writer())
{
this.countryToShards.forEach((country, shardList) ->
{
final List<String> shardNames = shardList.stream()
.map(slippyTile -> slippyTile.getName()).collect(Collectors.toList());
writer.writeLine(String.format("%s%s%s", country, DELIMITER, shardNames)
.replace("[", "").replace("]", ""));
});
}
catch (final Exception e)
{
throw new CoreException("Error while writing CountryToShardListCache to file!", e);
}
}

@Override
protected int onRun(final CommandMap command)
{
final StringList countries = (StringList) command.get(COUNTRIES);
final CountryBoundaryMap boundaries = (CountryBoundaryMap) command.get(BOUNDARIES);
final File output = (File) command.get(OUTPUT);
final Sharding sharding = (Sharding) command.get(SHARDING);
CountryToShardListCache cache = null;
if (countries != null)
{
cache = new CountryToShardListCache(boundaries, countries, sharding);
}
else
{
cache = new CountryToShardListCache(boundaries, sharding);
}
logger.info("Saving file to {}", output);
cache.save(output);
return 0;
}

@Override
protected SwitchList switches()
{
return new SwitchList().with(COUNTRIES, BOUNDARIES, OUTPUT, SHARDING);
}
}
64 changes: 64 additions & 0 deletions src/test/java/org/openstreetmap/atlas/geography/boundary/CountryToShardListCacheTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package org.openstreetmap.atlas.geography.boundary;

import java.util.List;

import org.junit.Assert;
import org.junit.Test;
import org.openstreetmap.atlas.geography.sharding.Sharding;
import org.openstreetmap.atlas.geography.sharding.SlippyTile;
import org.openstreetmap.atlas.streaming.resource.ByteArrayResource;
import org.openstreetmap.atlas.streaming.resource.InputStreamResource;
import org.openstreetmap.atlas.streaming.resource.Resource;
import org.openstreetmap.atlas.utilities.collections.StringList;

/**
* @author james-gage
*/
public class CountryToShardListCacheTest
{
private final Resource countryToShardList = new InputStreamResource(
() -> CountryToShardListCacheTest.class.getResourceAsStream("countryToShardList.txt"));

@Test
public void testGetShardNamesForCountry()
{
final CountryToShardListCache cache = new CountryToShardListCache(this.countryToShardList);
// test that the right DMA shards are returned
final List<SlippyTile> dMAShards = cache.getShardsForCountry("DMA");
Assert.assertEquals(
"[[SlippyTile: zoom = 9, x = 168, y = 233], [SlippyTile: zoom = 9, x = 169, y = 233], "
+ "[SlippyTile: zoom = 9, x = 168, y = 234], [SlippyTile: zoom = 10, x = 338, y = 468]]",
dMAShards.toString());
// test that asking for an invalid country code doesn't break anything
final List<SlippyTile> noShards = cache.getShardsForCountry("XXX");
Assert.assertTrue(noShards.isEmpty());
}

@Test
public void testSaveWhenBuildingFromFile()
{
final CountryToShardListCache cache = new CountryToShardListCache(this.countryToShardList);
// test that you get the same file back that was used to initialize the cache
final ByteArrayResource output = new ByteArrayResource();
cache.save(output);
Assert.assertEquals(this.countryToShardList.all(), output.all());
}

@Test
public void testSaveWhenBuildingFromScratch()
{
final StringList countries = new StringList();
countries.add("DMA");
final CountryBoundaryMap boundaries = CountryBoundaryMap
.fromPlainText(new InputStreamResource(() -> CountryToShardListCacheTest.class
.getResourceAsStream("DMA_boundary.txt")));
final Sharding sharding = Sharding.forString("dynamic@" + CountryToShardListCacheTest.class
.getResource("tree-6-14-100000.txt.gz").getPath());
// test building the cache from scratch
final CountryToShardListCache cache = new CountryToShardListCache(boundaries, countries,
sharding);
final ByteArrayResource output = new ByteArrayResource();
cache.save(output);
Assert.assertEquals("DMA||9-168-233, 9-169-233, 9-168-234, 10-338-468", output.all());
}
}
1 change: 1 addition & 0 deletions src/test/resources/org/openstreetmap/atlas/geography/boundary/countryToShardList.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
DMA||9-168-233, 9-169-233, 9-168-234, 10-338-468

0 comments on commit 85caca6

Please sign in to comment.