Skip to content

Commit

Permalink
File extension is now preserved by the temp file caching strategy (#225)
Browse files Browse the repository at this point in the history 
* File extension is now preserved by the temp file caching strategy

* checkstyle

* Shuffling some resources around

* PR comments and removed multithreading test.

* Removed a log4j file
  • Loading branch information
@lucaspcram @matthieun
lucaspcram authored and matthieun committed Sep 26, 2018
1 parent 47c6e7f commit b8da42c
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 131 deletions.
34 changes: 33 additions & 1 deletion .../openstreetmap/atlas/utilities/caching/strategies/SystemTemporaryFileCachingStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class SystemTemporaryFileCachingStrategy extends AbstractCachingStrategy
{
private static final Logger logger = LoggerFactory
.getLogger(SystemTemporaryFileCachingStrategy.class);
private static final String FILE_EXTENSION_DOT = ".";
private static final String PROPERTY_LOCAL_TEMPORARY_DIRECTORY = "java.io.tmpdir";
private static final String TEMPORARY_DIRECTORY_STRING = System
.getProperty(PROPERTY_LOCAL_TEMPORARY_DIRECTORY);
Expand All @@ -48,7 +49,17 @@ public Optional<Resource> attemptFetch(final URI resourceURI,
}

final Path temporaryDirectory = Paths.get(TEMPORARY_DIRECTORY_STRING);
final String cachedFileName = this.getUUIDForResourceURI(resourceURI).toString();
final Optional<String> resourceExtension = getFileExtensionFromURI(resourceURI);
final String cachedFileName;
if (resourceExtension.isPresent())
{
cachedFileName = this.getUUIDForResourceURI(resourceURI).toString() + FILE_EXTENSION_DOT
+ resourceExtension.get();
}
else
{
cachedFileName = this.getUUIDForResourceURI(resourceURI).toString();
}
final Path cachedFilePath = Paths.get(temporaryDirectory.toString(), cachedFileName);

final File cachedFile = new File(cachedFilePath.toString());
Expand Down Expand Up @@ -123,4 +134,25 @@ private void attemptToCacheFileLocally(final File cachedFile,
}
}
}

private Optional<String> getFileExtensionFromURI(final URI resourceURI)
{
final String asciiString = resourceURI.toASCIIString();
final int lastIndexOfDot = asciiString.lastIndexOf(FILE_EXTENSION_DOT);

if (lastIndexOfDot < 0)
{
return Optional.empty();
}

final String extension = asciiString.substring(lastIndexOfDot + 1);
if (extension.isEmpty())
{
return Optional.empty();
}
else
{
return Optional.of(extension);
}
}
}
202 changes: 72 additions & 130 deletions src/test/java/org/openstreetmap/atlas/utilities/caching/CachingTests.java
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,18 @@
package org.openstreetmap.atlas.utilities.caching;

import java.net.URI;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
import java.net.URISyntaxException;

import org.junit.Assert;
import org.junit.Test;
import org.openstreetmap.atlas.exception.CoreException;
import org.openstreetmap.atlas.streaming.resource.ByteArrayResource;
import org.openstreetmap.atlas.streaming.resource.File;
import org.openstreetmap.atlas.streaming.resource.InputStreamResource;
import org.openstreetmap.atlas.streaming.resource.Resource;
import org.openstreetmap.atlas.utilities.caching.strategies.ByteArrayCachingStrategy;
import org.openstreetmap.atlas.utilities.caching.strategies.CachingStrategy;
import org.openstreetmap.atlas.utilities.caching.strategies.NoCachingStrategy;
import org.openstreetmap.atlas.utilities.caching.strategies.SystemTemporaryFileCachingStrategy;
import org.openstreetmap.atlas.utilities.scalars.Duration;
import org.openstreetmap.atlas.utilities.time.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -25,59 +21,10 @@
*/
public class CachingTests
{
/**
* @author lcram
*/
private class CacheTask implements Runnable
{
private final ConcurrentResourceCache cache;
private final int size;

CacheTask(final ConcurrentResourceCache cache, final int size)
{
this.cache = cache;
this.size = size;
}

@Override
public void run()
{
final Resource resource1 = new File(LOCAL_TEST_FILE.toString());
final Resource resource2 = new File(LOCAL_TEST_FILE_2.toString());
final Random random = new Random();
for (int i = 0; i < this.size; i++)
{
if (random.nextBoolean())
{
compareFetchedResource(LOCAL_TEST_FILE_URI, resource1);
}
else
{
compareFetchedResource(LOCAL_TEST_FILE_2_URI, resource2);
}
}
}

private void compareFetchedResource(final URI resourceURI, final Resource goldenResource)
{
final Resource resource = this.cache.get(resourceURI).get();
if (!resource.readAndClose().equals(goldenResource.readAndClose()))
{
throw new CoreException("Unexpected resource discrepancy");
}
}
}

private static final int TASK_SIZE = 10000;

private static final Logger logger = LoggerFactory.getLogger(CachingTests.class);
private static final Path LOCAL_TEST_FILE = Paths.get("src/test/resources/log4j.properties")
.toAbsolutePath();
private static final URI LOCAL_TEST_FILE_URI = LOCAL_TEST_FILE.toUri();
private static final Path LOCAL_TEST_FILE_2 = Paths
.get("src/test/resources/org/openstreetmap/atlas/utilities/configuration/feature.json")
.toAbsolutePath();
private static final URI LOCAL_TEST_FILE_2_URI = LOCAL_TEST_FILE_2.toUri();

private static final String FEATURE_JSON = "feature.json";
private static final String FILE_NO_EXTENSION = "fileNoExt";

@Test
public void testBaseCacheWithByteArrayStrategy()
Expand All @@ -101,85 +48,37 @@ public void testBaseCacheWithNoStrategy()
public void testLocalFileInMemoryCache()
{
final LocalFileInMemoryCache cache = new LocalFileInMemoryCache();
URI resourceUri;
try
{
resourceUri = CachingTests.class.getResource(FEATURE_JSON).toURI();
}
catch (final URISyntaxException exception)
{
logger.error("{}", exception);
resourceUri = null;
Assert.fail();
}

// read the contents of the file
final ByteArrayResource originalFileBytes = new ByteArrayResource();
originalFileBytes.copyFrom(new File(LOCAL_TEST_FILE.toString()));
originalFileBytes.copyFrom(new InputStreamResource(
() -> CachingTests.class.getResourceAsStream(FEATURE_JSON)));
final byte[] originalFileBytesArray = originalFileBytes.readBytesAndClose();

// read contents of the file with cache, this will incur a cache miss
final ByteArrayResource fileBytesCacheMiss = new ByteArrayResource();
fileBytesCacheMiss.copyFrom(cache.get(LOCAL_TEST_FILE.toString()).get());
fileBytesCacheMiss.copyFrom(cache.get(resourceUri).get());
final byte[] fileBytesCacheMissArray = fileBytesCacheMiss.readBytesAndClose();
Assert.assertArrayEquals(originalFileBytesArray, fileBytesCacheMissArray);

// read contents again, this time with a cache hit
final ByteArrayResource fileBytesCacheHit = new ByteArrayResource();
fileBytesCacheHit.copyFrom(cache.get(LOCAL_TEST_FILE.toString()).get());
fileBytesCacheHit.copyFrom(cache.get(resourceUri).get());
final byte[] fileBytesCacheHitArray = fileBytesCacheHit.readBytesAndClose();
Assert.assertArrayEquals(originalFileBytesArray, fileBytesCacheHitArray);
}

@Test
public void testRepeatedCacheReads() throws InterruptedException
{
try
{
final ConcurrentResourceCache sharedCache1 = new ConcurrentResourceCache(
new ByteArrayCachingStrategy(), this::fetchLocalFileResource);
final Thread thread1 = new Thread(new CacheTask(sharedCache1, TASK_SIZE));
final Thread thread2 = new Thread(new CacheTask(sharedCache1, TASK_SIZE));
final Thread thread3 = new Thread(new CacheTask(sharedCache1, TASK_SIZE));
final Thread thread4 = new Thread(new CacheTask(sharedCache1, TASK_SIZE));
final Thread thread5 = new Thread(new CacheTask(sharedCache1, TASK_SIZE));
final Thread thread6 = new Thread(new CacheTask(sharedCache1, TASK_SIZE));
final Time time1 = Time.now();
thread1.start();
thread2.start();
thread3.start();
thread4.start();
thread5.start();
thread6.start();
thread1.join();
thread2.join();
thread3.join();
thread4.join();
thread5.join();
thread6.join();
final Duration duration1 = time1.elapsedSince();

final ConcurrentResourceCache sharedCache2 = new ConcurrentResourceCache(
new NoCachingStrategy(), this::fetchLocalFileResource);
final Thread thread7 = new Thread(new CacheTask(sharedCache2, TASK_SIZE));
final Thread thread8 = new Thread(new CacheTask(sharedCache2, TASK_SIZE));
final Thread thread9 = new Thread(new CacheTask(sharedCache2, TASK_SIZE));
final Thread thread10 = new Thread(new CacheTask(sharedCache2, TASK_SIZE));
final Thread thread11 = new Thread(new CacheTask(sharedCache2, TASK_SIZE));
final Thread thread12 = new Thread(new CacheTask(sharedCache2, TASK_SIZE));
final Time time2 = Time.now();
thread7.start();
thread8.start();
thread9.start();
thread10.start();
thread11.start();
thread12.start();
thread7.join();
thread8.join();
thread9.join();
thread10.join();
thread11.join();
thread12.join();
final Duration duration2 = time2.elapsedSince();

logger.info("File in memory ELAPSED: {}", duration1);
logger.info("No strategy ELAPSED: {}", duration2);
}
catch (final CoreException exception)
{
Assert.fail("A multithreaded discrepancy test failed");
}
}

private Resource fetchLocalFileResource(final URI resourceURI)
{
final String filePath = resourceURI.getPath();
Expand All @@ -193,21 +92,64 @@ private void testBaseCacheWithGivenStrategy(final CachingStrategy strategy)
final ConcurrentResourceCache resourceCache = new ConcurrentResourceCache(strategy,
this::fetchLocalFileResource);

URI resourceUri;
try
{
resourceUri = CachingTests.class.getResource(FEATURE_JSON).toURI();
}
catch (final URISyntaxException exception)
{
logger.error("{}", exception);
resourceUri = null;
Assert.fail();
}

URI resourceUri2;
try
{
resourceUri2 = CachingTests.class.getResource(FILE_NO_EXTENSION).toURI();
}
catch (final URISyntaxException exception)
{
logger.error("{}", exception);
resourceUri2 = null;
Assert.fail();
}

// read the contents of the file
final ByteArrayResource originalFileBytes = new ByteArrayResource();
originalFileBytes.copyFrom(new File(LOCAL_TEST_FILE.toString()));
final byte[] originalFileBytesArray = originalFileBytes.readBytesAndClose();
ByteArrayResource originalFileBytes = new ByteArrayResource();
originalFileBytes.copyFrom(new InputStreamResource(
() -> CachingTests.class.getResourceAsStream(FEATURE_JSON)));
byte[] originalFileBytesArray = originalFileBytes.readBytesAndClose();

// read contents of the file with cache, this will incur a cache miss
final ByteArrayResource fileBytesCacheMiss = new ByteArrayResource();
fileBytesCacheMiss.copyFrom(resourceCache.get(LOCAL_TEST_FILE_URI).get());
final byte[] fileBytesCacheMissArray = fileBytesCacheMiss.readBytesAndClose();
ByteArrayResource fileBytesCacheMiss = new ByteArrayResource();
fileBytesCacheMiss.copyFrom(resourceCache.get(resourceUri).get());
byte[] fileBytesCacheMissArray = fileBytesCacheMiss.readBytesAndClose();
Assert.assertArrayEquals(originalFileBytesArray, fileBytesCacheMissArray);

// read contents again, this time with a cache hit
final ByteArrayResource fileBytesCacheHit = new ByteArrayResource();
fileBytesCacheHit.copyFrom(resourceCache.get(LOCAL_TEST_FILE_URI).get());
final byte[] fileBytesCacheHitArray = fileBytesCacheHit.readBytesAndClose();
ByteArrayResource fileBytesCacheHit = new ByteArrayResource();
fileBytesCacheHit.copyFrom(resourceCache.get(resourceUri).get());
byte[] fileBytesCacheHitArray = fileBytesCacheHit.readBytesAndClose();
Assert.assertArrayEquals(originalFileBytesArray, fileBytesCacheHitArray);

// read the contents of the file
originalFileBytes = new ByteArrayResource();
originalFileBytes.copyFrom(new InputStreamResource(
() -> CachingTests.class.getResourceAsStream(FILE_NO_EXTENSION)));
originalFileBytesArray = originalFileBytes.readBytesAndClose();

// read contents of the file with cache, this will incur a cache miss
fileBytesCacheMiss = new ByteArrayResource();
fileBytesCacheMiss.copyFrom(resourceCache.get(resourceUri2).get());
fileBytesCacheMissArray = fileBytesCacheMiss.readBytesAndClose();
Assert.assertArrayEquals(originalFileBytesArray, fileBytesCacheMissArray);

// read contents again, this time with a cache hit
fileBytesCacheHit = new ByteArrayResource();
fileBytesCacheHit.copyFrom(resourceCache.get(resourceUri2).get());
fileBytesCacheHitArray = fileBytesCacheHit.readBytesAndClose();
Assert.assertArrayEquals(originalFileBytesArray, fileBytesCacheHitArray);
}
}
10 changes: 10 additions & 0 deletions src/test/resources/org/openstreetmap/atlas/utilities/caching/feature.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"feature": {
"XYZ": {
"range": {
"max": 40.0,
"min": 20.0
}
}
}
}
1 change: 1 addition & 0 deletions src/test/resources/org/openstreetmap/atlas/utilities/caching/fileNoExt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a test text file with no file extension

0 comments on commit b8da42c

Please sign in to comment.