diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index 83a5a51b0..c316b69a1 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-bigquery.git", - "sha": "2e08019e2312995995af3261350586fa59002b60" + "sha": "44c1a5d0083521b260bfa490c91b72051885eb4d" } }, { diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d028a6fe..f04d03d84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.133.0](https://www.github.com/googleapis/java-bigquery/compare/v1.132.1...v1.133.0) (2021-06-08) + + +### Features + +* add support for DecimalTargetTypes ([#1345](https://www.github.com/googleapis/java-bigquery/issues/1345)) ([ba528df](https://www.github.com/googleapis/java-bigquery/commit/ba528df03def71907e2811cf267718f090605d95)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-bigtable to v1.26.1 ([#1360](https://www.github.com/googleapis/java-bigquery/issues/1360)) ([bf55699](https://www.github.com/googleapis/java-bigquery/commit/bf55699d849de7e873577de04e44fbfe0f078ab1)) + ### [1.132.1](https://www.github.com/googleapis/java-bigquery/compare/v1.132.0...v1.132.1) (2021-06-07) diff --git a/README.md b/README.md index 4e0289083..a15eb421a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google com.google.cloud libraries-bom - 20.5.0 + 20.6.0 pom import @@ -40,25 +40,25 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 1.132.0 + 1.132.1 ``` If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:20.5.0') +implementation platform('com.google.cloud:libraries-bom:20.6.0') compile 'com.google.cloud:google-cloud-bigquery' ``` If you are using Gradle without BOM, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-bigquery:1.132.0' +compile 'com.google.cloud:google-cloud-bigquery:1.132.1' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "1.132.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "1.132.1" ``` ## Authentication diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 9137e7d53..67260984d 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 1.132.1 + 1.133.0 diff --git a/google-cloud-bigquery/clirr-ignored-differences.xml b/google-cloud-bigquery/clirr-ignored-differences.xml index d8b1ba813..2801782f4 100644 --- a/google-cloud-bigquery/clirr-ignored-differences.xml +++ b/google-cloud-bigquery/clirr-ignored-differences.xml @@ -4,32 +4,12 @@ 7013 - com/google/cloud/bigquery/MaterializedViewDefinition - com.google.cloud.bigquery.Clustering getClustering() + com/google/cloud/bigquery/ExternalTableDefinition + com.google.common.collect.ImmutableList getDecimalTargetTypes() 7013 - com/google/cloud/bigquery/MaterializedViewDefinition - com.google.cloud.bigquery.RangePartitioning getRangePartitioning() - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition - com.google.cloud.bigquery.TimePartitioning getTimePartitioning() - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition$Builder - com.google.cloud.bigquery.MaterializedViewDefinition$Builder setClustering(com.google.cloud.bigquery.Clustering) - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition$Builder - com.google.cloud.bigquery.MaterializedViewDefinition$Builder setRangePartitioning(com.google.cloud.bigquery.RangePartitioning) - - - 7013 - com/google/cloud/bigquery/MaterializedViewDefinition$Builder - com.google.cloud.bigquery.MaterializedViewDefinition$Builder setTimePartitioning(com.google.cloud.bigquery.TimePartitioning) + com/google/cloud/bigquery/ExternalTableDefinition$Builder + com.google.cloud.bigquery.ExternalTableDefinition$Builder setDecimalTargetTypes(java.util.List) \ No newline at end of file diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 4b88f59cc..d7e5074bc 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 1.132.1 + 1.133.0 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 1.132.1 + 1.133.0 google-cloud-bigquery diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java index 1d2cfa293..2b68a7ff2 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExternalTableDefinition.java @@ -93,6 +93,17 @@ public Builder setFormatOptions(FormatOptions formatOptions) { return setFormatOptionsInner(formatOptions); } + /** + * Defines the list of possible SQL data types to which the source decimal values are converted. + * This list and the precision and the scale parameters of the decimal field determine the + * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in + * the specified list and if it supports the precision and the scale. STRING supports all + * precision and scale values. + * + * @param decimalTargetTypes decimalTargetType or {@code null} for none + */ + public abstract Builder setDecimalTargetTypes(List decimalTargetTypes); + abstract Builder setFormatOptionsInner(FormatOptions formatOptions); /** @@ -229,6 +240,9 @@ public F getFormatOptions() { @Nullable abstract FormatOptions getFormatOptionsInner(); + @Nullable + public abstract ImmutableList getDecimalTargetTypes(); + /** * [Experimental] Returns whether automatic detection of schema and format options should be * performed. @@ -283,6 +297,9 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toExternalDataC if (getSourceUris() != null) { externalConfigurationPb.setSourceUris(getSourceUris()); } + if (getDecimalTargetTypes() != null) { + externalConfigurationPb.setDecimalTargetTypes(getDecimalTargetTypes()); + } if (getFormatOptions() != null && FormatOptions.CSV.equals(getFormatOptions().getType())) { externalConfigurationPb.setCsvOptions(((CsvOptions) getFormatOptions()).toPb()); } @@ -430,6 +447,10 @@ static ExternalTableDefinition fromPb(Table tablePb) { if (externalDataConfiguration.getSourceUris() != null) { builder.setSourceUris(ImmutableList.copyOf(externalDataConfiguration.getSourceUris())); } + if (externalDataConfiguration.getDecimalTargetTypes() != null) { + builder.setDecimalTargetTypes( + ImmutableList.copyOf(externalDataConfiguration.getDecimalTargetTypes())); + } if (externalDataConfiguration.getSourceFormat() != null) { builder.setFormatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat())); } @@ -469,6 +490,9 @@ static ExternalTableDefinition fromExternalDataConfiguration( if (externalDataConfiguration.getSourceUris() != null) { builder.setSourceUris(externalDataConfiguration.getSourceUris()); } + if (externalDataConfiguration.getDecimalTargetTypes() != null) { + builder.setDecimalTargetTypes(externalDataConfiguration.getDecimalTargetTypes()); + } if (externalDataConfiguration.getSchema() != null) { builder.setSchema(Schema.fromPb(externalDataConfiguration.getSchema())); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java index a317f1285..2f625acf5 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java @@ -38,6 +38,7 @@ public final class LoadJobConfiguration extends JobConfiguration implements Load private final List sourceUris; private final TableId destinationTable; + private final List decimalTargetTypes; private final EncryptionConfiguration destinationEncryptionConfiguration; private final JobInfo.CreateDisposition createDisposition; private final JobInfo.WriteDisposition writeDisposition; @@ -61,6 +62,7 @@ public static final class Builder extends JobConfiguration.Builder sourceUris; private TableId destinationTable; + private List decimalTargetTypes; private EncryptionConfiguration destinationEncryptionConfiguration; private JobInfo.CreateDisposition createDisposition; private JobInfo.WriteDisposition writeDisposition; @@ -87,6 +89,7 @@ private Builder() { private Builder(LoadJobConfiguration loadConfiguration) { this(); this.destinationTable = loadConfiguration.destinationTable; + this.decimalTargetTypes = loadConfiguration.decimalTargetTypes; this.createDisposition = loadConfiguration.createDisposition; this.writeDisposition = loadConfiguration.writeDisposition; this.formatOptions = loadConfiguration.formatOptions; @@ -112,6 +115,9 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur this(); JobConfigurationLoad loadConfigurationPb = configurationPb.getLoad(); this.destinationTable = TableId.fromPb(loadConfigurationPb.getDestinationTable()); + if (loadConfigurationPb.getDecimalTargetTypes() != null) { + this.decimalTargetTypes = ImmutableList.copyOf(loadConfigurationPb.getDecimalTargetTypes()); + } if (loadConfigurationPb.getCreateDisposition() != null) { this.createDisposition = JobInfo.CreateDisposition.valueOf(loadConfigurationPb.getCreateDisposition()); @@ -278,6 +284,20 @@ public Builder setSourceUris(List sourceUris) { return this; } + /** + * Defines the list of possible SQL data types to which the source decimal values are converted. + * This list and the precision and the scale parameters of the decimal field determine the + * target type. In the order of NUMERIC, BIGNUMERIC, and STRING, a type is picked if it is in + * the specified list and if it supports the precision and the scale. STRING supports all + * precision and scale values. + * + * @param decimalTargetTypes decimalTargetType or {@code null} for none + */ + public Builder setDecimalTargetTypes(List decimalTargetTypes) { + this.decimalTargetTypes = decimalTargetTypes; + return this; + } + public Builder setAutodetect(Boolean autodetect) { this.autodetect = autodetect; return this; @@ -341,6 +361,7 @@ private LoadJobConfiguration(Builder builder) { super(builder); this.sourceUris = builder.sourceUris; this.destinationTable = builder.destinationTable; + this.decimalTargetTypes = builder.decimalTargetTypes; this.createDisposition = builder.createDisposition; this.writeDisposition = builder.writeDisposition; this.formatOptions = builder.formatOptions; @@ -430,6 +451,10 @@ public List getSourceUris() { return sourceUris; } + public List getDecimalTargetTypes() { + return decimalTargetTypes; + } + public Boolean getAutodetect() { return autodetect; } @@ -482,6 +507,7 @@ public Builder toBuilder() { ToStringHelper toStringHelper() { return super.toStringHelper() .add("destinationTable", destinationTable) + .add("decimalTargetTypes", decimalTargetTypes) .add("destinationEncryptionConfiguration", destinationEncryptionConfiguration) .add("createDisposition", createDisposition) .add("writeDisposition", writeDisposition) @@ -568,6 +594,9 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (sourceUris != null) { loadConfigurationPb.setSourceUris(ImmutableList.copyOf(sourceUris)); } + if (decimalTargetTypes != null) { + loadConfigurationPb.setDecimalTargetTypes(ImmutableList.copyOf(decimalTargetTypes)); + } if (schemaUpdateOptions != null) { ImmutableList.Builder schemaUpdateOptionsBuilder = new ImmutableList.Builder<>(); for (JobInfo.SchemaUpdateOption schemaUpdateOption : schemaUpdateOptions) { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java index 23a095cb6..38dcd2714 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExternalTableDefinitionTest.java @@ -27,6 +27,8 @@ public class ExternalTableDefinitionTest { private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final List DECIMAL_TARGET_TYPES = + ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"); private static final Field FIELD_SCHEMA1 = Field.newBuilder("StringField", LegacySQLTypeName.STRING) .setMode(Field.Mode.NULLABLE) @@ -56,6 +58,7 @@ public class ExternalTableDefinitionTest { .build(); private static final ExternalTableDefinition EXTERNAL_TABLE_DEFINITION = ExternalTableDefinition.newBuilder(SOURCE_URIS, TABLE_SCHEMA, CSV_OPTIONS) + .setDecimalTargetTypes(DECIMAL_TARGET_TYPES) .setCompression(COMPRESSION) .setConnectionId(CONNECTION_ID) .setIgnoreUnknownValues(IGNORE_UNKNOWN_VALUES) @@ -111,6 +114,7 @@ public void testBuilder() { assertEquals(MAX_BAD_RECORDS, EXTERNAL_TABLE_DEFINITION.getMaxBadRecords()); assertEquals(TABLE_SCHEMA, EXTERNAL_TABLE_DEFINITION.getSchema()); assertEquals(SOURCE_URIS, EXTERNAL_TABLE_DEFINITION.getSourceUris()); + assertEquals(DECIMAL_TARGET_TYPES, EXTERNAL_TABLE_DEFINITION.getDecimalTargetTypes()); assertEquals(AUTODETECT, EXTERNAL_TABLE_DEFINITION.getAutodetect()); assertEquals(HIVE_PARTITIONING_OPTIONS, EXTERNAL_TABLE_DEFINITION.getHivePartitioningOptions()); assertNotEquals(EXTERNAL_TABLE_DEFINITION, TableDefinition.Type.EXTERNAL); @@ -130,6 +134,7 @@ public void testToAndFromPb() { private void compareExternalTableDefinition( ExternalTableDefinition expected, ExternalTableDefinition value) { assertEquals(expected, value); + assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes()); assertEquals(expected.getCompression(), value.getCompression()); assertEquals(expected.getConnectionId(), value.getConnectionId()); assertEquals(expected.getFormatOptions(), value.getFormatOptions()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java index 9f42d62b7..a2f164f8a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java @@ -50,6 +50,8 @@ public class LoadJobConfigurationTest { .setDescription("FieldDescription") .build(); private static final List SOURCE_URIS = ImmutableList.of("uri1", "uri2"); + private static final List DECIMAL_TARGET_TYPES = + ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"); private static final List SCHEMA_UPDATE_OPTIONS = ImmutableList.of(SchemaUpdateOption.ALLOW_FIELD_ADDITION); private static final Schema TABLE_SCHEMA = Schema.of(FIELD_SCHEMA); @@ -76,6 +78,7 @@ public class LoadJobConfigurationTest { .build(); private static final LoadJobConfiguration LOAD_CONFIGURATION_CSV = LoadJobConfiguration.newBuilder(TABLE_ID, SOURCE_URIS) + .setDecimalTargetTypes(DECIMAL_TARGET_TYPES) .setCreateDisposition(CREATE_DISPOSITION) .setWriteDisposition(WRITE_DISPOSITION) .setFormatOptions(CSV_OPTIONS) @@ -228,6 +231,7 @@ private void compareLoadJobConfiguration( assertEquals(expected.hashCode(), value.hashCode()); assertEquals(expected.toString(), value.toString()); assertEquals(expected.getDestinationTable(), value.getDestinationTable()); + assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes()); assertEquals(expected.getCreateDisposition(), value.getCreateDisposition()); assertEquals(expected.getWriteDisposition(), value.getWriteDisposition()); assertEquals(expected.getCsvOptions(), value.getCsvOptions()); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index eae807bf7..b151ad22c 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -2747,6 +2747,54 @@ public void testLoadJobWithRangePartitioning() throws InterruptedException { } } + @Test + public void testLoadJobWithDecimalTargetTypes() throws InterruptedException { + String tableName = "test_load_job_table_parquet_decimalTargetTypes"; + TableId destinationTable = TableId.of(DATASET, tableName); + String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet"; + try { + LoadJobConfiguration configuration = + LoadJobConfiguration.newBuilder(destinationTable, sourceUri, FormatOptions.parquet()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING")) + .build(); + Job job = bigquery.create(JobInfo.of(configuration)); + job = job.waitFor(); + assertNull(job.getStatus().getError()); + LoadJobConfiguration loadJobConfiguration = job.getConfiguration(); + assertEquals( + ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING"), + loadJobConfiguration.getDecimalTargetTypes()); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + assertEquals( + remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(), + "BIGNUMERIC"); + } finally { + bigquery.delete(destinationTable); + } + } + + @Test + public void testExternalTableWithDecimalTargetTypes() throws InterruptedException { + String tableName = "test_create_external_table_parquet_decimalTargetTypes"; + TableId destinationTable = TableId.of(DATASET, tableName); + String sourceUri = "gs://" + CLOUD_SAMPLES_DATA + "/bigquery/numeric/numeric_38_12.parquet"; + ExternalTableDefinition externalTableDefinition = + ExternalTableDefinition.newBuilder(sourceUri, FormatOptions.parquet()) + .setDecimalTargetTypes(ImmutableList.of("NUMERIC", "BIGNUMERIC", "STRING")) + .build(); + TableInfo tableInfo = TableInfo.of(destinationTable, externalTableDefinition); + Table createdTable = bigquery.create(tableInfo); + assertNotNull(createdTable); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + assertEquals( + remoteTable.getDefinition().getSchema().getFields().get(0).getType().toString(), + "BIGNUMERIC"); + assertTrue(remoteTable.delete()); + } + @Test public void testQueryJobWithDryRun() throws InterruptedException, TimeoutException { String tableName = "test_query_job_table"; diff --git a/pom.xml b/pom.xml index 50681b331..6422aadc6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 1.132.1 + 1.133.0 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -87,7 +87,7 @@ com.google.cloud google-cloud-bigquery - 1.132.1 + 1.133.0 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 16447990f..6608f9aa5 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 1.132.0 + 1.132.1 @@ -63,7 +63,7 @@ com.google.cloud google-cloud-bigtable - 1.26.0 + 1.26.1 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 32204cb42..e4b9f94c3 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 1.132.1 + 1.133.0 @@ -61,7 +61,7 @@ com.google.cloud google-cloud-bigtable - 1.26.0 + 1.26.1 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 75414a3a6..6682efa6c 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -79,7 +79,7 @@ com.google.cloud google-cloud-bigtable - 1.26.0 + 1.26.1 test diff --git a/versions.txt b/versions.txt index 0fef4ec61..4834b9a2a 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:1.132.1:1.132.1 \ No newline at end of file +google-cloud-bigquery:1.133.0:1.133.0 \ No newline at end of file