Skip to content

Commit

Permalink
Add a singleton for NoOpOrcWriterStats
Browse files Browse the repository at this point in the history
  • Loading branch information
sdruzkin authored and ARUNACHALAM THIRUPATHI committed Jun 30, 2022
1 parent 238f42b commit bff66c4
Show file tree
Hide file tree
Showing 20 changed files with 46 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import com.facebook.presto.common.io.DataSink;
import com.facebook.presto.common.type.Type;
import com.facebook.presto.orc.DefaultOrcWriterFlushPolicy;
import com.facebook.presto.orc.NoOpOrcWriterStats;
import com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode;
import com.facebook.presto.orc.OrcWriter;
import com.facebook.presto.orc.OrcWriterOptions;
Expand All @@ -34,6 +33,7 @@
import java.util.stream.IntStream;

import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.orc.metadata.CompressionKind.LZ4;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
Expand Down Expand Up @@ -100,7 +100,7 @@ private static OrcWriter createOrcFileWriter(DataSink sink, List<Type> types)
UTC,
false,
OrcWriteValidationMode.BOTH,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);
}
catch (NotSupportedException e) {
throw new PrestoException(NOT_SUPPORTED, e.getMessage(), e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import com.facebook.presto.hive.pagefile.PageWriter;
import com.facebook.presto.hive.parquet.ParquetPageSourceFactory;
import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory;
import com.facebook.presto.orc.NoOpOrcWriterStats;
import com.facebook.presto.orc.OrcWriter;
import com.facebook.presto.orc.OrcWriterOptions;
import com.facebook.presto.orc.StorageStripeMetadataSource;
Expand Down Expand Up @@ -92,6 +91,7 @@
import static com.facebook.presto.hive.pagefile.PageFileWriterFactory.createPagesSerdeForPageFile;
import static com.facebook.presto.hive.util.ConfigurationUtils.configureCompression;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.BOTH;
Expand Down Expand Up @@ -613,7 +613,7 @@ public PrestoOrcFormatWriter(File targetFile, List<String> columnNames, List<Typ
hiveStorageTimeZone,
false,
BOTH,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);
}

@Override
Expand Down Expand Up @@ -652,7 +652,7 @@ public PrestoDwrfFormatWriter(File targetFile, List<String> columnNames, List<Ty
hiveStorageTimeZone,
false,
BOTH,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
import static com.facebook.presto.iceberg.TypeConverter.toOrcType;
import static com.facebook.presto.iceberg.TypeConverter.toPrestoType;
import static com.facebook.presto.iceberg.util.PrimitiveTypeMapBuilder.makeTypeMap;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
import static com.google.common.collect.ImmutableList.toImmutableList;
Expand All @@ -78,7 +79,7 @@ public class IcebergFileWriterFactory
private final TypeManager typeManager;
private final FileFormatDataSourceStats readStats;
private final NodeVersion nodeVersion;
private final NoOpOrcWriterStats orcWriterStats = new NoOpOrcWriterStats();
private final NoOpOrcWriterStats orcWriterStats = NOOP_WRITER_STATS;
private final OrcFileWriterConfig orcFileWriterConfig;
private final DwrfEncryptionProvider dwrfEncryptionProvider;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
public class NoOpOrcWriterStats
implements WriterStats
{
public static final NoOpOrcWriterStats NOOP_WRITER_STATS = new NoOpOrcWriterStats();

@Override
public void recordStripeWritten(
int stripeMinBytes,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@

import static com.facebook.presto.common.type.IntegerType.INTEGER;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT;
import static com.facebook.presto.orc.NoopOrcLocalMemoryContext.NOOP_ORC_LOCAL_MEMORY_CONTEXT;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
Expand Down Expand Up @@ -203,7 +204,7 @@ private static TempFile writeOrcFile(boolean cacheEnabled, DwrfStripeCacheMode c
HIVE_STORAGE_TIME_ZONE,
true,
BOTH,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);

// write 4 stripes with 100 values each
int count = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature;
import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE;
import static com.facebook.presto.orc.OrcWriteValidation.OrcWriteValidationMode.BOTH;
Expand Down Expand Up @@ -117,7 +118,7 @@ private void doWrite(BenchmarkData data)
HIVE_STORAGE_TIME_ZONE,
false,
BOTH,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);

for (Block block : data.blocks) {
writer.write(new Page(block));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
import static com.facebook.presto.common.type.TinyintType.TINYINT;
import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE;
Expand Down Expand Up @@ -267,7 +268,7 @@ else if (typeSignature.startsWith("varchar")) {
}

// Use writeOrcColumnsPresto so that orcType and varchar length can be written in file footer
writeOrcColumnsPresto(orcFile, ORC_12, NONE, Optional.empty(), Collections.nCopies(channelCount, type), values, new NoOpOrcWriterStats());
writeOrcColumnsPresto(orcFile, ORC_12, NONE, Optional.empty(), Collections.nCopies(channelCount, type), values, NOOP_WRITER_STATS);
}

@TearDown
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@
import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager;
import static com.facebook.presto.orc.AbstractTestOrcReader.intsBetween;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT;
import static com.facebook.presto.orc.NoopOrcLocalMemoryContext.NOOP_ORC_LOCAL_MEMORY_CONTEXT;
import static com.facebook.presto.orc.OrcDecompressor.createOrcDecompressor;
Expand Down Expand Up @@ -635,7 +636,7 @@ private void assertRoundTrip(List<Type> writeTypes, List<Type> readTypes, List<L
assertEquals(writeTypes.size(), readValues.size());

AtomicLong totalSize = new AtomicLong(0);
WriterStats stats = new NoOpOrcWriterStats();
WriterStats stats = NOOP_WRITER_STATS;

Set<Integer> flattenedColumns = ImmutableSet.of();
if (flattenAllColumns) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static com.facebook.presto.orc.AbstractOrcRecordReader.getDecryptionKeyMetadata;
import static com.facebook.presto.orc.AbstractTestOrcReader.intsBetween;
import static com.facebook.presto.orc.DwrfEncryptionInfo.createNodeToGroupMap;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE;
Expand Down Expand Up @@ -617,7 +618,7 @@ private static void testDecryptionRoundTrip(
throws Exception
{
try (TempFile tempFile = new TempFile()) {
writeOrcColumnsPresto(tempFile.getFile(), OrcTester.Format.DWRF, ZSTD, dwrfWriterEncryption, types, writtenValues, new NoOpOrcWriterStats());
writeOrcColumnsPresto(tempFile.getFile(), OrcTester.Format.DWRF, ZSTD, dwrfWriterEncryption, types, writtenValues, NOOP_WRITER_STATS);

assertFileContentsPresto(
types,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import static com.facebook.presto.common.type.IntegerType.INTEGER;
import static com.facebook.presto.common.type.StandardTypes.ARRAY;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.orc.OrcReader.INITIAL_BATCH_SIZE;
Expand Down Expand Up @@ -824,7 +825,7 @@ private List<StripeFooter> testDictionary(Type type, OrcEncoding encoding, OrcWr
{
List<Type> types = ImmutableList.of(type);
try (TempFile tempFile = new TempFile()) {
OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, orcWriterOptions, new NoOpOrcWriterStats());
OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, orcWriterOptions, NOOP_WRITER_STATS);

int index = 0;
int batchId = 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import static com.facebook.presto.common.type.TinyintType.TINYINT;
import static com.facebook.presto.common.type.VarbinaryType.VARBINARY;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcTester.arrayType;
import static com.facebook.presto.orc.OrcTester.createOrcWriter;
import static com.facebook.presto.orc.OrcTester.mapType;
Expand Down Expand Up @@ -241,7 +242,7 @@ public void testMaxKeyLimit()
Optional.empty(),
ImmutableList.of(mapType),
writerOptions,
new NoOpOrcWriterStats())) {
NOOP_WRITER_STATS)) {
// write a block with 2 keys
orcWriter.write(createMapPageForKeyLimitTest(mapType, maxFlattenedMapKeyCount - 1));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.stream.IntStream;

import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcTester.Format.DWRF;
import static com.facebook.presto.orc.OrcTester.Format.ORC_11;
import static com.facebook.presto.orc.OrcTester.Format.ORC_12;
Expand Down Expand Up @@ -80,7 +81,7 @@ public void testEmptyStrings()

for (CompressionKind compression : compressions) {
TempFile tempFile = new TempFile();
writeOrcColumnsPresto(tempFile.getFile(), format, compression, Optional.empty(), types, values, new NoOpOrcWriterStats());
writeOrcColumnsPresto(tempFile.getFile(), format, compression, Optional.empty(), types, values, NOOP_WRITER_STATS);

OrcPredicate orcPredicate = createOrcPredicate(types, values, DWRF, false);
Map<Integer, Type> includedColumns = IntStream.range(0, types.size())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import static com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE;
Expand Down Expand Up @@ -147,7 +148,7 @@ private void testStreamOrder(OrcEncoding encoding, CompressionKind kind, Optiona
HIVE_STORAGE_TIME_ZONE,
true,
validationMode,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);

// write down some data with unsorted streams
String[] data = new String[] {"a", "bbbbb", "ccc", "dd", "eeee"};
Expand Down Expand Up @@ -209,7 +210,7 @@ public void testVerifyNoIllegalStateException()
HIVE_STORAGE_TIME_ZONE,
false,
null,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);

int entries = 65536;
BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, entries);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import static com.facebook.presto.common.type.TinyintType.TINYINT;
import static com.facebook.presto.common.type.VarbinaryType.VARBINARY;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcReader.MAX_BATCH_SIZE;
import static com.facebook.presto.orc.OrcTester.Format.DWRF;
import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE;
Expand Down Expand Up @@ -953,7 +954,7 @@ private void testMemoryTracking(CompressionKind compression, long lowerRetainedM
List<String> varcharDictionaryValues = newArrayList(limit(cycle(ImmutableList.of("apple", "apple pie", "apple\uD835\uDC03", "apple\uFFFD")), NUM_ROWS));
List<List<?>> values = ImmutableList.of(intValues, varcharDirectValues, varcharDictionaryValues);

writeOrcColumnsPresto(tempFile.getFile(), DWRF, compression, Optional.empty(), types, values, new NoOpOrcWriterStats());
writeOrcColumnsPresto(tempFile.getFile(), DWRF, compression, Optional.empty(), types, values, NOOP_WRITER_STATS);

OrcPredicate orcPredicate = createOrcPredicate(types, values, DWRF, false);
Map<Integer, Type> includedColumns = IntStream.range(0, types.size())
Expand Down Expand Up @@ -1015,7 +1016,7 @@ public void testOutputNotRequired()
List<String> varcharDirectValues = newArrayList(limit(cycle(ImmutableList.of("A", "B", "C")), NUM_ROWS));
List<List<?>> values = ImmutableList.of(varcharDirectValues, varcharDirectValues);

writeOrcColumnsPresto(tempFile.getFile(), DWRF, NONE, Optional.empty(), types, values, new NoOpOrcWriterStats());
writeOrcColumnsPresto(tempFile.getFile(), DWRF, NONE, Optional.empty(), types, values, NOOP_WRITER_STATS);

OrcPredicate orcPredicate = createOrcPredicate(types, values, DWRF, false);
Map<Subfield, TupleDomainFilter> filters = ImmutableMap.of(new Subfield("c"), stringIn(true, "A", "B", "C")); //ImmutableMap.of(1, stringIn(true, "10", "11"));
Expand Down Expand Up @@ -1094,7 +1095,7 @@ public void testAdaptiveBatchSizes()
}
}

writeOrcColumnsPresto(tempFile.getFile(), DWRF, NONE, Optional.empty(), types, ImmutableList.of(values), new NoOpOrcWriterStats());
writeOrcColumnsPresto(tempFile.getFile(), DWRF, NONE, Optional.empty(), types, ImmutableList.of(values), NOOP_WRITER_STATS);

try (OrcSelectiveRecordReader recordReader = createCustomOrcSelectiveRecordReader(tempFile, OrcEncoding.DWRF, OrcPredicate.TRUE, type, MAX_BATCH_SIZE, false, false)) {
assertEquals(recordReader.getFileRowCount(), rowCount);
Expand Down Expand Up @@ -1131,7 +1132,7 @@ public void testHiddenConstantColumns()
List<List<?>> values = ImmutableList.of(ImmutableList.of(1L, 2L));

TempFile tempFile = new TempFile();
writeOrcColumnsPresto(tempFile.getFile(), DWRF, ZSTD, Optional.empty(), types, values, new NoOpOrcWriterStats());
writeOrcColumnsPresto(tempFile.getFile(), DWRF, ZSTD, Optional.empty(), types, values, NOOP_WRITER_STATS);

// Hidden columns like partition columns use negative indices (-13).
int hiddenColumnIndex = -13;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager;
import static com.facebook.presto.orc.DwrfEncryptionProvider.NO_ENCRYPTION;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.NoopOrcAggregatedMemoryContext.NOOP_ORC_AGGREGATED_MEMORY_CONTEXT;
import static com.facebook.presto.orc.OrcEncoding.ORC;
import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE;
Expand Down Expand Up @@ -243,7 +244,7 @@ private void write(TempFile tempFile, Type writerType, List<String> data)
HIVE_STORAGE_TIME_ZONE,
true,
BOTH,
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);

// write down some data with unsorted streams
Block[] fieldBlocks = new Block[data.size()];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

import static com.facebook.presto.common.type.TimestampType.TIMESTAMP;
import static com.facebook.presto.common.type.TimestampType.TIMESTAMP_MICROSECONDS;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcTester.Format.DWRF;
import static com.facebook.presto.orc.OrcTester.Format.ORC_11;
import static com.facebook.presto.orc.OrcTester.Format.ORC_12;
Expand Down Expand Up @@ -189,7 +190,7 @@ private void testPrestoRoundTrip(Type writeType, List<SqlTimestamp> writeValues,
Optional.empty(),
ImmutableList.of(writeType),
ImmutableList.of(writeValues),
new NoOpOrcWriterStats());
NOOP_WRITER_STATS);

assertFileContentsPresto(
ImmutableList.of(readType),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import com.facebook.presto.orc.DefaultOrcWriterFlushPolicy;
import com.facebook.presto.orc.DwrfEncryptionInfo;
import com.facebook.presto.orc.FileOrcDataSource;
import com.facebook.presto.orc.NoOpOrcWriterStats;
import com.facebook.presto.orc.OrcEncoding;
import com.facebook.presto.orc.OrcTester;
import com.facebook.presto.orc.OrcWriter;
Expand Down Expand Up @@ -55,6 +54,7 @@
import static com.facebook.presto.common.type.TinyintType.TINYINT;
import static com.facebook.presto.common.type.VarbinaryType.VARBINARY;
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
import static com.facebook.presto.orc.NoOpOrcWriterStats.NOOP_WRITER_STATS;
import static com.facebook.presto.orc.OrcEncoding.DWRF;
import static com.facebook.presto.orc.OrcTester.HIVE_STORAGE_TIME_ZONE;
import static com.facebook.presto.orc.OrcTester.createOrcWriter;
Expand Down Expand Up @@ -312,7 +312,7 @@ public void testFileMetadataRawSize()

for (OrcEncoding encoding : OrcEncoding.values()) {
try (TempFile tempFile = new TempFile()) {
OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, writerOptions, new NoOpOrcWriterStats());
OrcWriter writer = createOrcWriter(tempFile.getFile(), encoding, ZSTD, Optional.empty(), types, writerOptions, NOOP_WRITER_STATS);
for (int i = 0; i < numBlocksPerFile; i++) {
writer.write(new Page(blocks));
}
Expand Down
Loading

0 comments on commit bff66c4

Please sign in to comment.