Skip to content

Commit

Permalink
Close down OrcReaderOptions c-tor for good
Browse files Browse the repository at this point in the history
  • Loading branch information
@sdruzkin
sdruzkin authored and ARUNACHALAM THIRUPATHI committed Jun 28, 2022
1 parent 5c03da3 commit 1290033
Show file tree
Hide file tree
Showing 14 changed files with 86 additions and 75 deletions.
11 changes: 6 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/orc/DwrfBatchPageSourceFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,12 @@ public Optional<? extends ConnectorPageSource> createPageSource(
orcFileTailSource,
stripeMetadataSourceFactory,
hiveFileContext,
new OrcReaderOptions(
getOrcMaxMergeDistance(session),
getOrcTinyStripeThreshold(session),
getOrcMaxReadBlockSize(session),
isOrcZstdJniDecompressionEnabled(session)),
OrcReaderOptions.builder()
.withMaxMergeDistance(getOrcMaxMergeDistance(session))
.withTinyStripeThreshold(getOrcTinyStripeThreshold(session))
.withMaxBlockSize(getOrcMaxReadBlockSize(session))
.withZstdJniDecompressionEnabled(isOrcZstdJniDecompressionEnabled(session))
.build(),
encryptionInformation,
dwrfEncryptionProvider));
}
Expand Down
11 changes: 6 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcBatchPageSourceFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,11 +186,12 @@ public Optional<? extends ConnectorPageSource> createPageSource(
orcFileTailSource,
stripeMetadataSourceFactory,
hiveFileContext,
new OrcReaderOptions(
getOrcMaxMergeDistance(session),
getOrcTinyStripeThreshold(session),
getOrcMaxReadBlockSize(session),
isOrcZstdJniDecompressionEnabled(session)),
OrcReaderOptions.builder()
.withMaxMergeDistance(getOrcMaxMergeDistance(session))
.withTinyStripeThreshold(getOrcTinyStripeThreshold(session))
.withMaxBlockSize(getOrcMaxReadBlockSize(session))
.withZstdJniDecompressionEnabled(isOrcZstdJniDecompressionEnabled(session))
.build(),
encryptionInformation,
NO_ENCRYPTION));
}
Expand Down
12 changes: 7 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/orc/OrcSelectivePageSourceFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,11 +298,13 @@ public static ConnectorPageSource createOrcPageSource(
DataSize streamBufferSize = getOrcStreamBufferSize(session);
DataSize tinyStripeThreshold = getOrcTinyStripeThreshold(session);
DataSize maxReadBlockSize = getOrcMaxReadBlockSize(session);
OrcReaderOptions orcReaderOptions = new OrcReaderOptions(maxMergeDistance,
tinyStripeThreshold,
maxReadBlockSize,
isOrcZstdJniDecompressionEnabled(session),
appendRowNumberEnabled);
OrcReaderOptions orcReaderOptions = OrcReaderOptions.builder()
.withMaxMergeDistance(maxMergeDistance)
.withTinyStripeThreshold(tinyStripeThreshold)
.withMaxBlockSize(maxReadBlockSize)
.withZstdJniDecompressionEnabled(isOrcZstdJniDecompressionEnabled(session))
.withAppendRowNumber(appendRowNumberEnabled)
.build();
boolean lazyReadSmallRanges = getOrcLazyReadSmallRanges(session);

OrcDataSource orcDataSource;
Expand Down
10 changes: 5 additions & 5 deletions presto-hive/src/main/java/com/facebook/presto/hive/util/TempFileReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@ public TempFileReader(List<Type> types, OrcDataSource dataSource)
new StorageOrcFileTailSource(),
new StorageStripeMetadataSource(),
new HiveOrcAggregatedMemoryContext(),
new OrcReaderOptions(
new DataSize(1, MEGABYTE),
new DataSize(8, MEGABYTE),
new DataSize(16, MEGABYTE),
false),
OrcReaderOptions.builder()
.withMaxMergeDistance(new DataSize(1, MEGABYTE))
.withTinyStripeThreshold(new DataSize(8, MEGABYTE))
.withMaxBlockSize(new DataSize(16, MEGABYTE))
.build(),
false,
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down
11 changes: 6 additions & 5 deletions presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergPageSourceProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,11 +241,12 @@ private ConnectorPageSource createDataPageSource(
fileFormatDataSourceStats,
false);
case ORC:
OrcReaderOptions readerOptions = new OrcReaderOptions(
getOrcMaxMergeDistance(session),
getOrcTinyStripeThreshold(session),
getOrcMaxReadBlockSize(session),
isOrcZstdJniDecompressionEnabled(session));
OrcReaderOptions readerOptions = OrcReaderOptions.builder()
.withMaxMergeDistance(getOrcMaxMergeDistance(session))
.withTinyStripeThreshold(getOrcTinyStripeThreshold(session))
.withMaxBlockSize(getOrcMaxReadBlockSize(session))
.withZstdJniDecompressionEnabled(isOrcZstdJniDecompressionEnabled(session))
.build();

// TODO: Implement EncryptionInformation in IcebergSplit instead of Optional.empty()
return createBatchOrcPageSource(
Expand Down
19 changes: 1 addition & 18 deletions presto-orc/src/main/java/com/facebook/presto/orc/OrcReaderOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,7 @@ public class OrcReaderOptions
// if the option is set to true, OrcSelectiveReader will append a row number block at the end of the page
private final boolean appendRowNumber;

public OrcReaderOptions(DataSize maxMergeDistance,
DataSize tinyStripeThreshold,
DataSize maxBlockSize,
boolean zstdJniDecompressionEnabled)
{
this(maxMergeDistance, tinyStripeThreshold, maxBlockSize, zstdJniDecompressionEnabled, false, false);
}

public OrcReaderOptions(DataSize maxMergeDistance,
DataSize tinyStripeThreshold,
DataSize maxBlockSize,
boolean zstdJniDecompressionEnabled,
boolean appendRowNumber)
{
this(maxMergeDistance, tinyStripeThreshold, maxBlockSize, zstdJniDecompressionEnabled, false, appendRowNumber);
}

public OrcReaderOptions(
private OrcReaderOptions(
DataSize maxMergeDistance,
DataSize tinyStripeThreshold,
DataSize maxBlockSize,
Expand Down
6 changes: 5 additions & 1 deletion presto-orc/src/main/java/com/facebook/presto/orc/OrcWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -846,7 +846,11 @@ public void validate(OrcDataSource input)
types,
hiveStorageTimeZone,
orcEncoding,
new OrcReaderOptions(new DataSize(1, MEGABYTE), new DataSize(8, MEGABYTE), new DataSize(16, MEGABYTE), false),
OrcReaderOptions.builder()
.withMaxMergeDistance(new DataSize(1, MEGABYTE))
.withTinyStripeThreshold(new DataSize(8, MEGABYTE))
.withMaxBlockSize(new DataSize(16, MEGABYTE))
.build(),
dwrfEncryptionProvider,
DwrfKeyProvider.of(intermediateKeyMetadata.build()));
}
Expand Down
7 changes: 6 additions & 1 deletion presto-orc/src/test/java/com/facebook/presto/orc/TestCachingOrcDataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import static com.facebook.presto.orc.metadata.CompressionKind.NONE;
import static com.facebook.presto.orc.metadata.CompressionKind.ZLIB;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static io.airlift.units.DataSize.Unit.MEGABYTE;
import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
Expand Down Expand Up @@ -241,7 +242,11 @@ public void doIntegration(TestingOrcDataSource orcDataSource, DataSize maxMergeD
new StorageOrcFileTailSource(),
new StorageStripeMetadataSource(),
NOOP_ORC_AGGREGATED_MEMORY_CONTEXT,
new OrcReaderOptions(maxMergeDistance, tinyStripeThreshold, new DataSize(1, Unit.MEGABYTE), false),
OrcReaderOptions.builder()
.withMaxMergeDistance(maxMergeDistance)
.withTinyStripeThreshold(tinyStripeThreshold)
.withMaxBlockSize(new DataSize(1, MEGABYTE))
.build(),
false,
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down
12 changes: 5 additions & 7 deletions presto-orc/src/test/java/com/facebook/presto/orc/TestDecryption.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -487,13 +487,11 @@ public void testSkipFirstStripe()
new StorageOrcFileTailSource(),
new StorageStripeMetadataSource(),
NOOP_ORC_AGGREGATED_MEMORY_CONTEXT,
new OrcReaderOptions(
new DataSize(1, MEGABYTE),
new DataSize(1, MEGABYTE),
MAX_BLOCK_SIZE,
false,
false,
false),
OrcReaderOptions.builder()
.withMaxMergeDistance(new DataSize(1, MEGABYTE))
.withTinyStripeThreshold(new DataSize(1, MEGABYTE))
.withMaxBlockSize(MAX_BLOCK_SIZE)
.build(),
false,
new DwrfEncryptionProvider(new UnsupportedEncryptionLibrary(), new TestingPlainKeyEncryptionLibrary()),
DwrfKeyProvider.of(ImmutableMap.of(0, Slices.utf8Slice("key"))),
Expand Down
10 changes: 5 additions & 5 deletions presto-orc/src/test/java/com/facebook/presto/orc/TestOrcRecordReaderDwrfStripeCaching.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,11 +140,11 @@ private void assertFileContentCachingEnabled(File orcFile, List<DiskRange> forbi
DwrfAwareStripeMetadataSourceFactory dwrfAwareFactory = new DwrfAwareStripeMetadataSourceFactory(delegateSourceFactory);

// set zeroes to avoid file caching and merging of small disk ranges
OrcReaderOptions orcReaderOptions = new OrcReaderOptions(
new DataSize(0, MEGABYTE),
new DataSize(0, MEGABYTE),
new DataSize(1, MEGABYTE),
false);
OrcReaderOptions orcReaderOptions = OrcReaderOptions.builder()
.withMaxMergeDistance(new DataSize(0, MEGABYTE))
.withTinyStripeThreshold(new DataSize(0, MEGABYTE))
.withMaxBlockSize(new DataSize(1, MEGABYTE))
.build();

OrcReader orcReader = new OrcReader(
orcDataSource,
Expand Down
10 changes: 5 additions & 5 deletions presto-orc/src/test/java/com/facebook/presto/orc/TestStructBatchStreamReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,11 +278,11 @@ private RowBlock read(TempFile tempFile, Type readerType)
new StorageOrcFileTailSource(),
new StorageStripeMetadataSource(),
NOOP_ORC_AGGREGATED_MEMORY_CONTEXT,
new OrcReaderOptions(
dataSize,
dataSize,
dataSize,
false),
OrcReaderOptions.builder()
.withMaxMergeDistance(dataSize)
.withTinyStripeThreshold(dataSize)
.withMaxBlockSize(dataSize)
.build(),
false,
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down
7 changes: 6 additions & 1 deletion presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcFileRewriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,12 @@ public OrcFileInfo rewrite(FileSystem fileSystem, Map<String, Type> allColumnTyp
orcFileTailSource,
stripeMetadataSourceFactory,
new RaptorOrcAggregatedMemoryContext(),
new OrcReaderOptions(readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, readerAttributes.isZstdJniDecompressionEnabled()),
OrcReaderOptions.builder()
.withMaxMergeDistance(readerAttributes.getMaxMergeDistance())
.withTinyStripeThreshold(readerAttributes.getTinyStripeThreshold())
.withMaxBlockSize(HUGE_MAX_READ_BLOCK_SIZE)
.withZstdJniDecompressionEnabled(readerAttributes.isZstdJniDecompressionEnabled())
.build(),
false,
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down
25 changes: 18 additions & 7 deletions presto-raptor/src/main/java/com/facebook/presto/raptor/storage/OrcStorageManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,12 @@ public ConnectorPageSource getPageSource(
orcFileTailSource,
stripeMetadataSourceFactory,
new RaptorOrcAggregatedMemoryContext(),
new OrcReaderOptions(readerAttributes.getMaxMergeDistance(), readerAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, readerAttributes.isZstdJniDecompressionEnabled()),
OrcReaderOptions.builder()
.withMaxMergeDistance(readerAttributes.getMaxMergeDistance())
.withTinyStripeThreshold(readerAttributes.getTinyStripeThreshold())
.withMaxBlockSize(HUGE_MAX_READ_BLOCK_SIZE)
.withZstdJniDecompressionEnabled(readerAttributes.isZstdJniDecompressionEnabled())
.build(),
hiveFileContext.isCacheable(),
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down Expand Up @@ -388,11 +393,12 @@ Optional<BitSet> getRowsFromUuid(FileSystem fileSystem, Optional<UUID> deltaShar
orcFileTailSource,
new StorageStripeMetadataSource(),
new RaptorOrcAggregatedMemoryContext(),
new OrcReaderOptions(
defaultReaderAttributes.getMaxMergeDistance(),
defaultReaderAttributes.getTinyStripeThreshold(),
HUGE_MAX_READ_BLOCK_SIZE,
defaultReaderAttributes.isZstdJniDecompressionEnabled()),
OrcReaderOptions.builder()
.withMaxMergeDistance(defaultReaderAttributes.getMaxMergeDistance())
.withTinyStripeThreshold(defaultReaderAttributes.getTinyStripeThreshold())
.withMaxBlockSize(HUGE_MAX_READ_BLOCK_SIZE)
.withZstdJniDecompressionEnabled(defaultReaderAttributes.isZstdJniDecompressionEnabled())
.build(),
false,
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down Expand Up @@ -558,7 +564,12 @@ private List<ColumnStats> computeShardStats(FileSystem fileSystem, Path file)
orcFileTailSource,
stripeMetadataSourceFactory,
new RaptorOrcAggregatedMemoryContext(),
new OrcReaderOptions(defaultReaderAttributes.getMaxMergeDistance(), defaultReaderAttributes.getTinyStripeThreshold(), HUGE_MAX_READ_BLOCK_SIZE, defaultReaderAttributes.isZstdJniDecompressionEnabled()),
OrcReaderOptions.builder()
.withMaxMergeDistance(defaultReaderAttributes.getMaxMergeDistance())
.withTinyStripeThreshold(defaultReaderAttributes.getTinyStripeThreshold())
.withMaxBlockSize(HUGE_MAX_READ_BLOCK_SIZE)
.withZstdJniDecompressionEnabled(defaultReaderAttributes.isZstdJniDecompressionEnabled())
.build(),
false,
NO_ENCRYPTION,
DwrfKeyProvider.EMPTY,
Expand Down
10 changes: 5 additions & 5 deletions presto-raptor/src/test/java/com/facebook/presto/raptor/storage/OrcTestingUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,10 +129,10 @@ public static FileWriter createFileWriter(List<Long> columnIds, List<Type> colum

public static OrcReaderOptions createDefaultTestConfig()
{
return new OrcReaderOptions(
new DataSize(1, MEGABYTE),
new DataSize(1, MEGABYTE),
new DataSize(1, MEGABYTE),
false);
return OrcReaderOptions.builder()
.withMaxMergeDistance(new DataSize(1, MEGABYTE))
.withTinyStripeThreshold(new DataSize(1, MEGABYTE))
.withMaxBlockSize(new DataSize(1, MEGABYTE))
.build();
}
}

0 comments on commit 1290033

Please sign in to comment.