diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 48ad11d44..aadf54f64 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:944c07e458ce227ca49a423affedf088e31e2ee70908dd21682238d58f1beb60 + digest: sha256:ad9cabee4c022f1aab04a71332369e0c23841062124818a4490f73337f790337 diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml index d4ca94189..57d2b0f83 100644 --- a/.github/release-trigger.yml +++ b/.github/release-trigger.yml @@ -1 +1,2 @@ enabled: true +multiScmName: java-bigquery diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index e20330c3c..f52533545 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.0" + value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.2" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index 0fd6ba2fa..44b100487 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.0" + value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.2" } env_vars: { diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f8d68714..cceeee3b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [2.27.0](https://github.com/googleapis/java-bigquery/compare/v2.26.1...v2.27.0) (2023-05-30) + + +### Features + +* Add support for session id on TableDataWriteChannel ([#2715](https://github.com/googleapis/java-bigquery/issues/2715)) ([42851d8](https://github.com/googleapis/java-bigquery/commit/42851d818ee825d7c4141d40d116e1da43c11f14)) + + +### Bug Fixes + +* Add support for repeated record query parameters ([#2698](https://github.com/googleapis/java-bigquery/issues/2698)) ([51aff50](https://github.com/googleapis/java-bigquery/commit/51aff502215d69bd0151030421cd18646c6ead36)) + + +### Dependencies + +* Update dependency com.google.api.grpc:proto-google-cloud-bigqueryconnection-v1 to v2.20.0 ([#2720](https://github.com/googleapis/java-bigquery/issues/2720)) ([4962cac](https://github.com/googleapis/java-bigquery/commit/4962cac8fb3fe8d77a136eaf1b579cd79304acfb)) +* Update dependency com.google.apis:google-api-services-bigquery to v2-rev20230506-2.0.0 ([#2707](https://github.com/googleapis/java-bigquery/issues/2707)) ([4d2ec07](https://github.com/googleapis/java-bigquery/commit/4d2ec0716287e9624949cbcdf6605c127c209be4)) +* Update dependency com.google.apis:google-api-services-bigquery to v2-rev20230520-2.0.0 ([#2723](https://github.com/googleapis/java-bigquery/issues/2723)) ([5c64797](https://github.com/googleapis/java-bigquery/commit/5c64797c603343408849535b2dbf8080cd11ca32)) +* Update dependency com.google.cloud:google-cloud-bigquerystorage-bom to v2.37.2 ([#2726](https://github.com/googleapis/java-bigquery/issues/2726)) ([052c47a](https://github.com/googleapis/java-bigquery/commit/052c47aa43b0f50414db3031914e8a775ae98925)) +* Update dependency com.google.cloud:google-cloud-datacatalog-bom to v1.24.0 ([#2721](https://github.com/googleapis/java-bigquery/issues/2721)) ([7c357fb](https://github.com/googleapis/java-bigquery/commit/7c357fb414d45fde734c09c88ee3023d8d8f5822)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.10.1 ([#2713](https://github.com/googleapis/java-bigquery/issues/2713)) ([744e83a](https://github.com/googleapis/java-bigquery/commit/744e83a3da5323bc2cff2bcc6368a3eec39f392e)) + ## [2.26.1](https://github.com/googleapis/java-bigquery/compare/v2.26.0...v2.26.1) (2023-05-16) diff --git a/README.md b/README.md index 9f50f9a51..d26c0dbcf 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google com.google.cloud libraries-bom - 26.14.0 + 26.15.0 pom import @@ -45,7 +45,7 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-bigquery - 2.26.0 + 2.26.1 ``` @@ -60,13 +60,13 @@ implementation 'com.google.cloud:google-cloud-bigquery' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.26.0' +implementation 'com.google.cloud:google-cloud-bigquery:2.26.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.26.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.26.1" ``` @@ -127,6 +127,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-bigquery/tree | Copy Multiple Tables | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java) | | Copy Table | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CopyTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CopyTable.java) | | Copy Table Cmek | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CopyTableCmek.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CopyTableCmek.java) | +| Create And Query Repeated Record Field | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java) | | Create Clustered Table | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateClusteredTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateClusteredTable.java) | | Create Dataset | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateDataset.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateDataset.java) | | Create Dataset Aws | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateDatasetAws.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateDatasetAws.java) | @@ -348,7 +349,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.26.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.26.1 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 977aaf3a4..1f8ff6a03 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -6,7 +6,7 @@ google-cloud-bigquery-parent com.google.cloud - 2.26.1 + 2.27.0 diff --git a/google-cloud-bigquery/pom.xml b/google-cloud-bigquery/pom.xml index 0bef4df31..32757e502 100644 --- a/google-cloud-bigquery/pom.xml +++ b/google-cloud-bigquery/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquery - 2.26.1 + 2.27.0 jar BigQuery https://github.com/googleapis/java-bigquery @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquery-parent - 2.26.1 + 2.27.0 google-cloud-bigquery diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java index b92cb734c..b21445d38 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java @@ -349,7 +349,11 @@ public static QueryParameterValue array(T[] array, Class clazz) { public static QueryParameterValue array(T[] array, StandardSQLTypeName type) { List listValues = new ArrayList<>(); for (T obj : array) { - listValues.add(QueryParameterValue.of(obj, type)); + if (type == StandardSQLTypeName.STRUCT) { + listValues.add((QueryParameterValue) obj); + } else { + listValues.add(QueryParameterValue.of(obj, type)); + } } return QueryParameterValue.newBuilder() .setArrayValues(listValues) @@ -522,9 +526,15 @@ QueryParameterType toTypePb() { QueryParameterType typePb = new QueryParameterType(); typePb.setType(getType().toString()); if (getArrayType() != null) { - QueryParameterType arrayTypePb = new QueryParameterType(); - arrayTypePb.setType(getArrayType().toString()); - typePb.setArrayType(arrayTypePb); + List values = getArrayValues(); + if (getArrayType() == StandardSQLTypeName.STRUCT && values != null && values.size() != 0) { + QueryParameterType structType = values.get(0).toTypePb(); + typePb.setArrayType(structType); + } else { + QueryParameterType arrayTypePb = new QueryParameterType(); + arrayTypePb.setType(getArrayType().toString()); + typePb.setArrayType(arrayTypePb); + } } if (getStructTypes() != null) { List structTypes = new ArrayList<>(); diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java index cdb9db904..114c6dadd 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/WriteChannelConfiguration.java @@ -25,6 +25,7 @@ import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import com.google.common.primitives.Ints; import java.io.Serializable; import java.util.List; @@ -56,9 +57,11 @@ public final class WriteChannelConfiguration implements LoadConfiguration, Seria private final Boolean useAvroLogicalTypes; private final Map labels; private List decimalTargetTypes; + private final List connectionProperties; - public static final class Builder implements LoadConfiguration.Builder { + private final Boolean createSession; + public static final class Builder implements LoadConfiguration.Builder { private TableId destinationTable; private CreateDisposition createDisposition; private WriteDisposition writeDisposition; @@ -75,6 +78,9 @@ public static final class Builder implements LoadConfiguration.Builder { private Boolean useAvroLogicalTypes; private Map labels; private List decimalTargetTypes; + private List connectionProperties; + + private Boolean createSession; private Builder() {} @@ -96,6 +102,8 @@ private Builder(WriteChannelConfiguration writeChannelConfiguration) { this.useAvroLogicalTypes = writeChannelConfiguration.useAvroLogicalTypes; this.labels = writeChannelConfiguration.labels; this.decimalTargetTypes = writeChannelConfiguration.decimalTargetTypes; + this.connectionProperties = writeChannelConfiguration.connectionProperties; + this.createSession = writeChannelConfiguration.createSession; } private Builder(com.google.api.services.bigquery.model.JobConfiguration configurationPb) { @@ -175,6 +183,13 @@ private Builder(com.google.api.services.bigquery.model.JobConfiguration configur if (loadConfigurationPb.getDecimalTargetTypes() != null) { this.decimalTargetTypes = loadConfigurationPb.getDecimalTargetTypes(); } + if (loadConfigurationPb.getConnectionProperties() != null) { + + this.connectionProperties = + Lists.transform( + loadConfigurationPb.getConnectionProperties(), ConnectionProperty.FROM_PB_FUNCTION); + } + createSession = loadConfigurationPb.getCreateSession(); } @Override @@ -274,6 +289,16 @@ public Builder setDecimalTargetTypes(List decimalTargetTypes) { return this; } + public Builder setConnectionProperties(List connectionProperties) { + this.connectionProperties = ImmutableList.copyOf(connectionProperties); + return this; + } + + public Builder setCreateSession(Boolean createSession) { + this.createSession = createSession; + return this; + } + @Override public WriteChannelConfiguration build() { return new WriteChannelConfiguration(this); @@ -297,6 +322,8 @@ protected WriteChannelConfiguration(Builder builder) { this.useAvroLogicalTypes = builder.useAvroLogicalTypes; this.labels = builder.labels; this.decimalTargetTypes = builder.decimalTargetTypes; + this.connectionProperties = builder.connectionProperties; + this.createSession = builder.createSession; } @Override @@ -390,6 +417,14 @@ public List getDecimalTargetTypes() { return decimalTargetTypes; } + public List getConnectionProperties() { + return connectionProperties; + } + + public Boolean getCreateSession() { + return createSession; + } + @Override public Builder toBuilder() { return new Builder(this); @@ -412,7 +447,9 @@ MoreObjects.ToStringHelper toStringHelper() { .add("clustering", clustering) .add("useAvroLogicalTypes", useAvroLogicalTypes) .add("labels", labels) - .add("decimalTargetTypes", decimalTargetTypes); + .add("decimalTargetTypes", decimalTargetTypes) + .add("connectionProperties", connectionProperties) + .add("createSession", createSession); } @Override @@ -444,7 +481,9 @@ public int hashCode() { clustering, useAvroLogicalTypes, labels, - decimalTargetTypes); + decimalTargetTypes, + connectionProperties, + createSession); } WriteChannelConfiguration setProjectId(String projectId) { @@ -519,6 +558,13 @@ com.google.api.services.bigquery.model.JobConfiguration toPb() { if (decimalTargetTypes != null) { loadConfigurationPb.setDecimalTargetTypes(decimalTargetTypes); } + if (connectionProperties != null) { + loadConfigurationPb.setConnectionProperties( + Lists.transform(connectionProperties, ConnectionProperty.TO_PB_FUNCTION)); + } + if (createSession != null) { + loadConfigurationPb.setCreateSession(createSession); + } jobConfiguration.setLoad(loadConfigurationPb); return jobConfiguration; } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java index 05920df23..0534865b2 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java @@ -28,6 +28,7 @@ import java.math.BigDecimal; import java.text.ParseException; import java.time.Period; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -563,6 +564,48 @@ public void testNestedStruct() { assertThat(nestedRecordField.getStructValues().size()).isEqualTo(structValue.size()); } + @Test + public void testStructArray() { + Boolean[] boolValues = new Boolean[] {true, false}; + Integer[] intValues = new Integer[] {15, 20}; + String[] stringValues = new String[] {"test-string", "test-string2"}; + List> fieldMaps = new ArrayList<>(); + List tuples = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + QueryParameterValue booleanField = QueryParameterValue.bool(boolValues[i]); + QueryParameterValue integerField = QueryParameterValue.int64(intValues[i]); + QueryParameterValue stringField = QueryParameterValue.string(stringValues[i]); + ImmutableMap fieldMap = + ImmutableMap.of( + "booleanField", + booleanField, + "integerField", + integerField, + "stringField", + stringField); + fieldMaps.add(fieldMap); + QueryParameterValue recordField = QueryParameterValue.struct(fieldMap); + tuples.add(recordField); + } + QueryParameterValue repeatedRecordField = + QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT); + com.google.api.services.bigquery.model.QueryParameterValue parameterValue = + repeatedRecordField.toValuePb(); + QueryParameterType parameterType = repeatedRecordField.toTypePb(); + QueryParameterValue queryParameterValue = + QueryParameterValue.fromPb(parameterValue, parameterType); + assertThat(queryParameterValue.getValue()).isNull(); + assertThat(queryParameterValue.getType()).isEqualTo(StandardSQLTypeName.ARRAY); + assertThat(queryParameterValue.getArrayType()).isEqualTo(StandardSQLTypeName.STRUCT); + assertThat(queryParameterValue.getArrayValues().size()).isEqualTo(2); + for (int i = 0; i < 2; i++) { + QueryParameterValue record = queryParameterValue.getArrayValues().get(i); + assertThat(record.getType()).isEqualTo(StandardSQLTypeName.STRUCT); + assertThat(record.getStructTypes()).isNotNull(); + assertThat(record.getStructValues()).isEqualTo(fieldMaps.get(i)); + } + } + private static void assertArrayDataEquals( String[] expectedValues, StandardSQLTypeName expectedType, diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java index 01c86e8fb..7b912ce2b 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/WriteChannelConfigurationTest.java @@ -61,6 +61,14 @@ public class WriteChannelConfigurationTest { ImmutableMap.of("test-job-name", "test-write-channel"); private static final List DECIMAL_TARGET_TYPES = ImmutableList.of("NUMERIC", "BIGNUMERIC"); + + private static final boolean CREATE_SESSION = true; + private static final String KEY = "session_id"; + private static final String VALUE = "session_id_1234567890"; + private static final ConnectionProperty CONNECTION_PROPERTY = + ConnectionProperty.newBuilder().setKey(KEY).setValue(VALUE).build(); + private static final List CONNECTION_PROPERTIES = + ImmutableList.of(CONNECTION_PROPERTY); private static final WriteChannelConfiguration LOAD_CONFIGURATION_CSV = WriteChannelConfiguration.newBuilder(TABLE_ID) .setCreateDisposition(CREATE_DISPOSITION) @@ -76,6 +84,8 @@ public class WriteChannelConfigurationTest { .setClustering(CLUSTERING) .setLabels(LABELS) .setDecimalTargetTypes(DECIMAL_TARGET_TYPES) + .setConnectionProperties(CONNECTION_PROPERTIES) + .setCreateSession(CREATE_SESSION) .build(); private static final DatastoreBackupOptions BACKUP_OPTIONS = @@ -232,5 +242,7 @@ private void compareLoadConfiguration( assertEquals(expected.getUseAvroLogicalTypes(), value.getUseAvroLogicalTypes()); assertEquals(expected.getLabels(), value.getLabels()); assertEquals(expected.getDecimalTargetTypes(), value.getDecimalTargetTypes()); + assertEquals(expected.getConnectionProperties(), value.getConnectionProperties()); + assertEquals(expected.getCreateSession(), value.getCreateSession()); } } diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index 6d61fd211..cdfcb8003 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -28,6 +28,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import com.google.api.client.util.IOUtils; import com.google.api.gax.paging.Page; import com.google.auth.oauth2.GoogleCredentials; import com.google.auth.oauth2.ServiceAccountCredentials; @@ -60,6 +61,7 @@ import com.google.cloud.bigquery.ConnectionProperty; import com.google.cloud.bigquery.ConnectionSettings; import com.google.cloud.bigquery.CopyJobConfiguration; +import com.google.cloud.bigquery.CsvOptions; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.DatasetId; import com.google.cloud.bigquery.DatasetInfo; @@ -141,9 +143,13 @@ import com.google.gson.JsonObject; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Path; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Time; @@ -156,6 +162,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -231,6 +238,31 @@ public class ITBigQueryTest { .setMode(Field.Mode.REQUIRED) .setDescription("RecordDescription") .build(); + + private static final Field REPEATED_RECORD_FIELD_SCHEMA = + Field.newBuilder( + "Addresses", + LegacySQLTypeName.RECORD, + Field.newBuilder("Status", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("Address", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("City", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("State", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("Zip", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("NumberOfYears", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build()) + .setMode(Field.Mode.REPEATED) + .build(); private static final Field INTEGER_FIELD_SCHEMA = Field.newBuilder("IntegerField", LegacySQLTypeName.INTEGER) .setMode(Field.Mode.NULLABLE) @@ -422,6 +454,18 @@ public class ITBigQueryTest { .setMode(Field.Mode.NULLABLE) .build()); + private static final Schema REPEATED_RECORD_TABLE_SCHEMA = + Schema.of( + Field.newBuilder("ID", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build(), + Field.newBuilder("FirstName", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("LastName", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("DOB", LegacySQLTypeName.DATE).setMode(Field.Mode.NULLABLE).build(), + REPEATED_RECORD_FIELD_SCHEMA); + private static final Schema SIMPLE_SCHEMA = Schema.of(STRING_FIELD_SCHEMA); private static final Schema QUERY_RESULT_SCHEMA = Schema.of( @@ -673,6 +717,36 @@ public class ITBigQueryTest { private static final List CONNECTION_PROPERTIES = ImmutableList.of(CONNECTION_PROPERTY); + private static final Field ID_SCHEMA = + Field.newBuilder("id", LegacySQLTypeName.STRING) + .setMode(Mode.REQUIRED) + .setDescription("id") + .build(); + private static final Field FIRST_NAME_SCHEMA = + Field.newBuilder("firstname", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("First Name") + .build(); + private static final Field LAST_NAME_SCHEMA = + Field.newBuilder("lastname", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("LAST NAME") + .build(); + private static final Field EMAIL_SCHEMA = + Field.newBuilder("email", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("email") + .build(); + private static final Field PROFESSION_SCHEMA = + Field.newBuilder("profession", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .setDescription("profession") + .build(); + private static final Schema SESSION_TABLE_SCHEMA = + Schema.of(ID_SCHEMA, FIRST_NAME_SCHEMA, LAST_NAME_SCHEMA, EMAIL_SCHEMA, PROFESSION_SCHEMA); + private static final Path csvPath = + FileSystems.getDefault().getPath("src/test/resources", "sessionTest.csv").toAbsolutePath(); + private static final Set PUBLIC_DATASETS = ImmutableSet.of("github_repos", "hacker_news", "noaa_gsod", "samples", "usa_names"); @@ -3695,6 +3769,80 @@ public void testQuerySessionSupport() throws InterruptedException { assertEquals(sessionId, statisticsWithSession.getSessionInfo().getSessionId()); } + @Test + public void testLoadSessionSupportWriteChannelConfiguration() throws InterruptedException { + TableId sessionTableId = TableId.of("_SESSION", "test_temp_destination_table_from_file"); + + WriteChannelConfiguration configuration = + WriteChannelConfiguration.newBuilder(sessionTableId) + .setFormatOptions(CsvOptions.newBuilder().setFieldDelimiter(",").build()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setSchema(SESSION_TABLE_SCHEMA) + .setCreateSession(true) + .build(); + String jobName = "jobId_" + UUID.randomUUID().toString(); + JobId jobId = JobId.newBuilder().setLocation("us").setJob(jobName).build(); + String sessionId; + + // Imports a local file into a table. + try (TableDataWriteChannel writer = bigquery.writer(jobId, configuration); + OutputStream stream = Channels.newOutputStream(writer)) { + InputStream inputStream = + ITBigQueryTest.class.getClassLoader().getResourceAsStream("sessionTest.csv"); + // Can use `Files.copy(csvPath, stream);` instead. + // Using IOUtils here because graalvm can't handle resource files. + IOUtils.copy(inputStream, stream); + + } catch (IOException e) { + throw new RuntimeException(e); + } + Job loadJob = bigquery.getJob(jobId); + Job completedJob = loadJob.waitFor(); + + assertNotNull(completedJob); + assertEquals(jobId.getJob(), completedJob.getJobId().getJob()); + JobStatistics.LoadStatistics statistics = completedJob.getStatistics(); + + sessionId = statistics.getSessionInfo().getSessionId(); + assertNotNull(sessionId); + + // Load job in the same session. + // Should load the data to a temp table. + ConnectionProperty sessionConnectionProperty = + ConnectionProperty.newBuilder().setKey("session_id").setValue(sessionId).build(); + WriteChannelConfiguration sessionConfiguration = + WriteChannelConfiguration.newBuilder(sessionTableId) + .setConnectionProperties(ImmutableList.of(sessionConnectionProperty)) + .setFormatOptions(CsvOptions.newBuilder().setFieldDelimiter(",").build()) + .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) + .setSchema(SESSION_TABLE_SCHEMA) + .build(); + String sessionJobName = "jobId_" + UUID.randomUUID().toString(); + JobId sessionJobId = JobId.newBuilder().setLocation("us").setJob(sessionJobName).build(); + try (TableDataWriteChannel writer = bigquery.writer(sessionJobId, sessionConfiguration); + OutputStream stream = Channels.newOutputStream(writer)) { + InputStream inputStream = + ITBigQueryTest.class.getClassLoader().getResourceAsStream("sessionTest.csv"); + IOUtils.copy(inputStream, stream); + } catch (IOException e) { + throw new RuntimeException(e); + } + Job queryJobWithSession = bigquery.getJob(sessionJobId); + queryJobWithSession = queryJobWithSession.waitFor(); + LoadStatistics statisticsWithSession = queryJobWithSession.getStatistics(); + assertNotNull(statisticsWithSession.getSessionInfo().getSessionId()); + + // Checking if the data loaded to the temp table in the session + String queryTempTable = "SELECT * FROM _SESSION.test_temp_destination_table_from_file;"; + QueryJobConfiguration queryJobConfigurationWithSession = + QueryJobConfiguration.newBuilder(queryTempTable) + .setConnectionProperties(ImmutableList.of(sessionConnectionProperty)) + .build(); + Job queryTempTableJob = bigquery.create(JobInfo.of(queryJobConfigurationWithSession)); + queryTempTableJob = queryTempTableJob.waitFor(); + assertNotNull(queryTempTableJob.getQueryResults()); + } + @Test public void testLoadSessionSupport() throws InterruptedException { // Start the session @@ -4062,6 +4210,214 @@ public void testStructNamedQueryParameters() throws InterruptedException { } } + @Test + public void testRepeatedRecordNamedQueryParameters() throws InterruptedException { + String[] stringValues = new String[] {"test-stringField", "test-stringField2"}; + List tuples = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + QueryParameterValue stringValue = QueryParameterValue.string(stringValues[i]); + Map struct = new HashMap<>(); + struct.put("stringField", stringValue); + QueryParameterValue recordValue = QueryParameterValue.struct(struct); + tuples.add(recordValue); + } + + QueryParameterValue repeatedRecord = + QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT); + String query = "SELECT @repeatedRecordField AS repeatedRecord"; + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DATASET) + .setUseLegacySql(false) + .addNamedParameter("repeatedRecordField", repeatedRecord) + .build(); + TableResult result = bigquery.query(config); + assertEquals(1, Iterables.size(result.getValues())); + + FieldList subSchema = result.getSchema().getFields().get("repeatedRecord").getSubFields(); + for (FieldValueList values : result.iterateAll()) { + for (FieldValue value : values) { + assertEquals(FieldValue.Attribute.REPEATED, value.getAttribute()); + assertEquals(2, value.getRepeatedValue().size()); + for (int i = 0; i < 2; i++) { + FieldValue record = value.getRepeatedValue().get(i); + assertEquals(FieldValue.Attribute.RECORD, record.getAttribute()); + FieldValueList recordValue = record.getRecordValue(); + assertEquals( + stringValues[i], + FieldValueList.of(recordValue, subSchema).get("stringField").getValue()); + } + } + } + } + + @Test + public void testUnnestRepeatedRecordNamedQueryParameter() throws InterruptedException { + Boolean[] boolValues = new Boolean[] {true, false}; + List tuples = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + QueryParameterValue boolValue = QueryParameterValue.bool(boolValues[i]); + Map struct = new HashMap<>(); + struct.put("boolField", boolValue); + QueryParameterValue recordValue = QueryParameterValue.struct(struct); + tuples.add(recordValue); + } + + QueryParameterValue repeatedRecord = + QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT); + String query = + "SELECT * FROM (SELECT STRUCT(" + + boolValues[0] + + " AS boolField) AS repeatedRecord) WHERE repeatedRecord IN UNNEST(@repeatedRecordField)"; + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DATASET) + .setUseLegacySql(false) + .addNamedParameter("repeatedRecordField", repeatedRecord) + .build(); + TableResult result = bigquery.query(config); + assertEquals(1, Iterables.size(result.getValues())); + + FieldList subSchema = result.getSchema().getFields().get("repeatedRecord").getSubFields(); + for (FieldValueList values : result.iterateAll()) { + for (FieldValue value : values) { + assertEquals(FieldValue.Attribute.RECORD, value.getAttribute()); + FieldValueList recordValue = value.getRecordValue(); + assertEquals( + boolValues[0], + FieldValueList.of(recordValue, subSchema).get("boolField").getBooleanValue()); + } + } + } + + @Test + public void testUnnestRepeatedRecordNamedQueryParameterFromDataset() throws InterruptedException { + TableId tableId = TableId.of(DATASET, "test_repeated_record_table"); + setUpRepeatedRecordTable(tableId); + + List tuples = new ArrayList<>(); + QueryParameterValue statusValue = QueryParameterValue.string("single"); + QueryParameterValue addressValue = QueryParameterValue.string("123 this lane"); + QueryParameterValue cityValue = QueryParameterValue.string("Toronto"); + QueryParameterValue stateValue = QueryParameterValue.string("ON"); + QueryParameterValue zipValue = QueryParameterValue.string("1h2j34"); + QueryParameterValue numberOfYearsValue = QueryParameterValue.string("3"); + + Map struct = new LinkedHashMap<>(); + struct.put("statusValue", statusValue); + struct.put("addressValue", addressValue); + struct.put("cityValue", cityValue); + struct.put("stateValue", stateValue); + struct.put("zipValue", zipValue); + struct.put("numberOfYearsValue", numberOfYearsValue); + QueryParameterValue recordValue = QueryParameterValue.struct(struct); + tuples.add(recordValue); + + QueryParameterValue repeatedRecord = + QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT); + + String query = + "SELECT * FROM " + + tableId.getTable() + + ", UNNEST(@repeatedRecord) AS TEMP where TEMP IN UNNEST(addresses);"; + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DATASET) + .setUseLegacySql(false) + .addNamedParameter("repeatedRecord", repeatedRecord) + .build(); + TableResult results = bigquery.query(queryConfig); + + assertEquals(1, Iterables.size(results.getValues())); + for (FieldValueList values : results.iterateAll()) { + assertEquals("1", values.get("ID").getStringValue()); + assertEquals("first_name1", values.get("FirstName").getStringValue()); + assertEquals(2, values.get("Addresses").getRecordValue().size()); + } + } + + private void setUpRepeatedRecordTable(TableId tableId) { + StandardTableDefinition tableDefinition = + StandardTableDefinition.of(REPEATED_RECORD_TABLE_SCHEMA); + TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); + bigquery.create(tableInfo); + + ImmutableMap.Builder builder1 = ImmutableMap.builder(); + builder1.put("ID", "1"); + builder1.put("FirstName", "first_name1"); + builder1.put("LastName", "last_name1"); + builder1.put("DOB", "1995-08-09"); + builder1.put( + "Addresses", + ImmutableList.of( + ImmutableMap.of( + "Status", "single", + "Address", "123 this lane", + "City", "Toronto", + "State", "ON", + "Zip", "1h2j34", + "NumberOfYears", "3"), + ImmutableMap.of( + "Status", "couple", + "Address", "345 that lane", + "City", "Maple", + "State", "ON", + "Zip", "1h2j34", + "NumberOfYears", "5"))); + + ImmutableMap.Builder builder2 = ImmutableMap.builder(); + builder2.put("ID", "2"); + builder2.put("FirstName", "first_name2"); + builder2.put("LastName", "last_name2"); + builder2.put("DOB", "1992-03-19"); + builder2.put( + "Addresses", + ImmutableList.of( + ImmutableMap.of( + "Status", "single", + "Address", "97 Kota lane", + "City", "Ottawa", + "State", "ON", + "Zip", "1h2j34", + "NumberOfYears", "3"), + ImmutableMap.of( + "Status", "couple", + "Address", "75 Malta lane", + "City", "Victoria", + "State", "AL", + "Zip", "1h2j34", + "NumberOfYears", "5"))); + + InsertAllRequest request = + InsertAllRequest.newBuilder(tableInfo.getTableId()) + .addRow(builder1.build()) + .addRow(builder2.build()) + .build(); + bigquery.insertAll(request); + } + + @Test + public void testEmptyRepeatedRecordNamedQueryParameters() throws InterruptedException { + QueryParameterValue[] tuples = {}; + + QueryParameterValue repeatedRecord = + QueryParameterValue.array(tuples, StandardSQLTypeName.STRUCT); + String query = + "SELECT * FROM (SELECT STRUCT(false AS boolField) AS repeatedRecord) WHERE repeatedRecord IN UNNEST(@repeatedRecordField)"; + QueryJobConfiguration config = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(DATASET) + .setUseLegacySql(false) + .addNamedParameter("repeatedRecordField", repeatedRecord) + .build(); + try { + bigquery.query(config); + fail("an empty array of struct query parameter shouldn't work with 'IN UNNEST'"); + } catch (BigQueryException e) { + // Nothing to do + } + } + @Test public void testStructQuery() throws InterruptedException { // query into a table diff --git a/google-cloud-bigquery/src/test/resources/META-INF/native-image/resource-config.json b/google-cloud-bigquery/src/test/resources/META-INF/native-image/resource-config.json index e00ed1f1c..97298417a 100644 --- a/google-cloud-bigquery/src/test/resources/META-INF/native-image/resource-config.json +++ b/google-cloud-bigquery/src/test/resources/META-INF/native-image/resource-config.json @@ -1,3 +1,4 @@ { - "resources":[{"pattern": ".*.csv"}] + "resources":[{"pattern": ".*.csv"}, + {"pattern": ".*src/test/resources/sessionTest.csv"}] } \ No newline at end of file diff --git a/google-cloud-bigquery/src/test/resources/sessionTest.csv b/google-cloud-bigquery/src/test/resources/sessionTest.csv new file mode 100644 index 000000000..f500c80c1 --- /dev/null +++ b/google-cloud-bigquery/src/test/resources/sessionTest.csv @@ -0,0 +1,51 @@ +id,firstname,lastname,email,profession +100,Rani,Merell,Rani.Merell@yopmail.com,firefighter +101,Goldie,Dex,Goldie.Dex@yopmail.com,developer +102,Cristabel,Munn,Cristabel.Munn@yopmail.com,developer +103,Genevra,Strephon,Genevra.Strephon@yopmail.com,firefighter +104,Augustine,Thema,Augustine.Thema@yopmail.com,doctor +105,Jemie,Gombach,Jemie.Gombach@yopmail.com,police officer +106,Maye,Stuart,Maye.Stuart@yopmail.com,developer +107,Ayn,Carmena,Ayn.Carmena@yopmail.com,worker +108,Gale,Celestine,Gale.Celestine@yopmail.com,doctor +109,Alex,Jerold,Alex.Jerold@yopmail.com,firefighter +110,Violet,Giule,Violet.Giule@yopmail.com,firefighter +111,Starla,Uird,Starla.Uird@yopmail.com,doctor +112,Tarra,Pelagias,Tarra.Pelagias@yopmail.com,police officer +113,Eugine,Deny,Eugine.Deny@yopmail.com,doctor +114,Shirlee,Ricarda,Shirlee.Ricarda@yopmail.com,doctor +115,Ariela,Penelopa,Ariela.Penelopa@yopmail.com,worker +116,Lelah,Astra,Lelah.Astra@yopmail.com,police officer +117,Debee,Deegan,Debee.Deegan@yopmail.com,developer +118,Pollyanna,Euridice,Pollyanna.Euridice@yopmail.com,worker +119,Cathie,Halsey,Cathie.Halsey@yopmail.com,firefighter +120,Rebeca,Quinn,Rebeca.Quinn@yopmail.com,doctor +121,Paulita,Arquit,Paulita.Arquit@yopmail.com,police officer +122,Rebeca,Emanuel,Rebeca.Emanuel@yopmail.com,firefighter +123,Tera,Ilka,Tera.Ilka@yopmail.com,firefighter +124,Orsola,Briney,Orsola.Briney@yopmail.com,doctor +125,Paulita,Wyn,Paulita.Wyn@yopmail.com,doctor +126,Constance,Christine,Constance.Christine@yopmail.com,firefighter +127,Claresta,Kinnard,Claresta.Kinnard@yopmail.com,developer +128,Leanna,Mendez,Leanna.Mendez@yopmail.com,developer +129,Corina,Chabot,Corina.Chabot@yopmail.com,developer +130,Romona,Audly,Romona.Audly@yopmail.com,worker +131,Cordi,Lynn,Cordi.Lynn@yopmail.com,firefighter +132,Sheree,Tyson,Sheree.Tyson@yopmail.com,worker +133,Jinny,Bevin,Jinny.Bevin@yopmail.com,police officer +134,Kassey,Havens,Kassey.Havens@yopmail.com,firefighter +135,Wanda,Thema,Wanda.Thema@yopmail.com,developer +136,Vita,Jagir,Vita.Jagir@yopmail.com,developer +137,Alie,Aprile,Alie.Aprile@yopmail.com,firefighter +138,Modestia,Jena,Modestia.Jena@yopmail.com,doctor +139,Cyndie,Pelagias,Cyndie.Pelagias@yopmail.com,worker +140,Ariela,Lilybelle,Ariela.Lilybelle@yopmail.com,firefighter +141,Jan,Parette,Jan.Parette@yopmail.com,firefighter +142,Merry,Horan,Merry.Horan@yopmail.com,developer +143,Katuscha,Candy,Katuscha.Candy@yopmail.com,police officer +144,Kerrin,Heisel,Kerrin.Heisel@yopmail.com,developer +145,Nollie,Magdalen,Nollie.Magdalen@yopmail.com,doctor +146,Karlee,Gordon,Karlee.Gordon@yopmail.com,developer +147,Dolli,Fadiman,Dolli.Fadiman@yopmail.com,firefighter +148,Leontine,Delp,Leontine.Delp@yopmail.com,worker +149,Ricky,Nadia,Ricky.Nadia@yopmail.com,doctor diff --git a/pom.xml b/pom.xml index e72403209..a7cedc04d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquery-parent pom - 2.26.1 + 2.27.0 BigQuery Parent https://github.com/googleapis/java-bigquery @@ -54,8 +54,8 @@ UTF-8 github google-cloud-bigquery-parent - v2-rev20230422-2.0.0 - 3.9.0 + v2-rev20230520-2.0.0 + 3.10.1 12.0.0 @@ -73,7 +73,7 @@ com.google.cloud google-cloud-bigquerystorage-bom - 2.37.0 + 2.37.2 pom import @@ -97,7 +97,7 @@ com.google.cloud google-cloud-datacatalog-bom - 1.23.0 + 1.24.0 pom import @@ -111,7 +111,7 @@ com.google.cloud google-cloud-bigquery - 2.26.1 + 2.27.0 @@ -137,7 +137,7 @@ com.google.truth truth - 1.1.3 + 1.1.4 test @@ -155,19 +155,19 @@ com.google.cloud google-cloud-storage - 2.22.2 + 2.22.3 test com.google.cloud google-cloud-bigqueryconnection - 2.19.0 + 2.20.0 test com.google.api.grpc proto-google-cloud-bigqueryconnection-v1 - 2.19.0 + 2.20.0 test @@ -197,7 +197,7 @@ org.apache.maven.plugins maven-project-info-reports-plugin - 3.4.3 + 3.4.4 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index c4f504c32..241d1b12b 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -45,7 +45,7 @@ com.google.cloud google-cloud-bigquery - 2.26.0 + 2.26.1 @@ -69,7 +69,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.19.0 + 2.20.0 test @@ -81,7 +81,7 @@ com.google.truth truth - 1.1.3 + 1.1.4 test diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 6cf142b53..d48ceb280 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -39,7 +39,7 @@ com.google.cloud libraries-bom - 26.14.0 + 26.15.0 pom import @@ -62,7 +62,7 @@ com.google.truth truth - 1.1.3 + 1.1.4 test diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index c0f3ac06b..eb260b372 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -44,7 +44,7 @@ com.google.cloud google-cloud-bigquery - 2.26.1 + 2.27.0 @@ -67,7 +67,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.19.0 + 2.20.0 test @@ -79,7 +79,7 @@ com.google.truth truth - 1.1.3 + 1.1.4 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 330260257..274942702 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -47,7 +47,7 @@ com.google.cloud libraries-bom - 26.14.0 + 26.15.0 pom import @@ -85,7 +85,7 @@ com.google.cloud google-cloud-bigqueryconnection - 2.19.0 + 2.20.0 test @@ -97,7 +97,7 @@ com.google.truth truth - 1.1.3 + 1.1.4 test diff --git a/samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java b/samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java new file mode 100644 index 000000000..2bb13eb12 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java @@ -0,0 +1,196 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +// [START bigquery_create_and_query_repeated_record] +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.QueryParameterValue; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.TableResult; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +// Create a table with a repeated record field and query it using an array of struct named parameter +public class CreateAndQueryRepeatedRecordField { + + private static final Field REPEATED_RECORD_FIELD_SCHEMA = + Field.newBuilder( + "Addresses", + LegacySQLTypeName.RECORD, + Field.newBuilder("Status", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("Address", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("City", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("State", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("Zip", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("NumberOfYears", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build()) + .setMode(Field.Mode.REPEATED) + .build(); + private static final Schema REPEATED_RECORD_TABLE_SCHEMA = + Schema.of( + Field.newBuilder("ID", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build(), + Field.newBuilder("FirstName", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("LastName", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("DOB", LegacySQLTypeName.DATE).setMode(Field.Mode.NULLABLE).build(), + REPEATED_RECORD_FIELD_SCHEMA); + + public static void main(String[] args) { + // TODO(developer): Replace these variables before running the sample. + String datasetName = "MY_DATASET_NAME"; + String tableName = "MY_TABLE_NAME"; + createAndQueryRepeatedRecordField(datasetName, tableName); + } + + public static void createAndQueryRepeatedRecordField(String datasetName, String tableName) { + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + TableId tableId = TableId.of(datasetName, tableName); + + // Create a table with a repeated record field + StandardTableDefinition tableDefinition = + StandardTableDefinition.of(REPEATED_RECORD_TABLE_SCHEMA); + TableInfo tableInfo = TableInfo.of(tableId, tableDefinition); + bigquery.create(tableInfo); + + // Insert some data + ImmutableMap.Builder builder1 = ImmutableMap.builder(); + builder1.put("ID", "1"); + builder1.put("FirstName", "first_name1"); + builder1.put("LastName", "last_name1"); + builder1.put("DOB", "1995-08-09"); + builder1.put( + "Addresses", + ImmutableList.of( + ImmutableMap.of( + "Status", "single", + "Address", "123 this lane", + "City", "Toronto", + "State", "ON", + "Zip", "1h2j34", + "NumberOfYears", "3"), + ImmutableMap.of( + "Status", "couple", + "Address", "345 that lane", + "City", "Maple", + "State", "ON", + "Zip", "1h2j34", + "NumberOfYears", "5"))); + + ImmutableMap.Builder builder2 = ImmutableMap.builder(); + builder2.put("ID", "2"); + builder2.put("FirstName", "first_name2"); + builder2.put("LastName", "last_name2"); + builder2.put("DOB", "1992-03-19"); + builder2.put( + "Addresses", + ImmutableList.of( + ImmutableMap.of( + "Status", "single", + "Address", "97 Kota lane", + "City", "Ottawa", + "State", "ON", + "Zip", "1h2j34", + "NumberOfYears", "3"), + ImmutableMap.of( + "Status", "couple", + "Address", "75 Malta lane", + "City", "Victoria", + "State", "AL", + "Zip", "1h2j34", + "NumberOfYears", "5"))); + + InsertAllRequest request = + InsertAllRequest.newBuilder(tableInfo.getTableId()) + .addRow(builder1.build()) + .addRow(builder2.build()) + .build(); + bigquery.insertAll(request); + + // Query using a named parameter + QueryParameterValue statusValue = QueryParameterValue.string("single"); + QueryParameterValue addressValue = QueryParameterValue.string("123 this lane"); + QueryParameterValue cityValue = QueryParameterValue.string("Toronto"); + QueryParameterValue stateValue = QueryParameterValue.string("ON"); + QueryParameterValue zipValue = QueryParameterValue.string("1h2j34"); + QueryParameterValue numberOfYearsValue = QueryParameterValue.string("3"); + + Map struct = new LinkedHashMap<>(); + struct.put("statusValue", statusValue); + struct.put("addressValue", addressValue); + struct.put("cityValue", cityValue); + struct.put("stateValue", stateValue); + struct.put("zipValue", zipValue); + struct.put("numberOfYearsValue", numberOfYearsValue); + QueryParameterValue recordValue = QueryParameterValue.struct(struct); + List tuples = new ArrayList<>(); + tuples.add(recordValue); + + QueryParameterValue repeatedRecord = + QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT); + + String query = + "SELECT * FROM " + + tableId.getTable() + + ", UNNEST(@repeatedRecord) AS TEMP where TEMP IN UNNEST(addresses);"; + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder(query) + .setDefaultDataset(datasetName) + .setUseLegacySql(false) + .addNamedParameter("repeatedRecord", repeatedRecord) + .build(); + TableResult results = bigquery.query(queryConfig); + results + .iterateAll() + .forEach(row -> row.forEach(val -> System.out.printf("%s\n", val.toString()))); + System.out.println("Query with Array of struct parameters performed successfully."); + } catch (BigQueryException | InterruptedException e) { + System.out.println("Query not performed \n" + e.toString()); + } + } +} +// [END bigquery_create_and_query_repeated_record] diff --git a/samples/snippets/src/main/java/com/example/bigquery/QueryWithArrayOfStructsNamedParameters.java b/samples/snippets/src/main/java/com/example/bigquery/QueryWithArrayOfStructsNamedParameters.java index 00cae0aa9..3a2074f9a 100644 --- a/samples/snippets/src/main/java/com/example/bigquery/QueryWithArrayOfStructsNamedParameters.java +++ b/samples/snippets/src/main/java/com/example/bigquery/QueryWithArrayOfStructsNamedParameters.java @@ -53,7 +53,7 @@ public static void queryWithArrayOfStructsNamedParameters() { .setUseLegacySql(false) .addNamedParameter( "arrayOfStructField", - QueryParameterValue.array(arrayOfStructs.toArray(), StandardSQLTypeName.STRING)) + QueryParameterValue.array(arrayOfStructs.toArray(), StandardSQLTypeName.STRUCT)) .build(); TableResult results = bigquery.query(queryConfig); results diff --git a/samples/snippets/src/test/java/com/example/bigquery/CreateAndQueryRepeatedRecordFieldIT.java b/samples/snippets/src/test/java/com/example/bigquery/CreateAndQueryRepeatedRecordFieldIT.java new file mode 100644 index 000000000..56ad47a5d --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigquery/CreateAndQueryRepeatedRecordFieldIT.java @@ -0,0 +1,79 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CreateAndQueryRepeatedRecordFieldIT { + + private final Logger log = Logger.getLogger(this.getClass().getName()); + private String tableName; + private ByteArrayOutputStream bout; + private PrintStream out; + private PrintStream originalPrintStream; + + private static final String BIGQUERY_DATASET_NAME = System.getenv("BIGQUERY_DATASET_NAME"); + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable " + varName + " is required to perform these tests.", + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("BIGQUERY_DATASET_NAME"); + } + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + originalPrintStream = System.out; + System.setOut(out); + tableName = "MY_TABLE_NAME_" + UUID.randomUUID().toString().replace("-", "_"); + } + + @After + public void tearDown() { + // Clean up + DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName); + // restores print statements in the original method + System.out.flush(); + System.setOut(originalPrintStream); + log.log(Level.INFO, "\n" + bout.toString()); + } + + @Test + public void testCreateAndQueryRepeatedRecordField() { + CreateAndQueryRepeatedRecordField.createAndQueryRepeatedRecordField( + BIGQUERY_DATASET_NAME, tableName); + assertThat(bout.toString()) + .contains("Query with Array of struct parameters performed successfully."); + } +} diff --git a/versions.txt b/versions.txt index c376a508f..2427ff336 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-bigquery:2.26.1:2.26.1 \ No newline at end of file +google-cloud-bigquery:2.27.0:2.27.0 \ No newline at end of file