diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0f0647b00e..4ca0036da3 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:e76136cc48f90aa19ba29cdfbd4002111467e44a1c9d905867d98dafafbd03bb + digest: sha256:27b1b1884dce60460d7521b23c2a73376cba90c0ef3d9f0d32e4bdb786959cfd diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index 1e30d1bd0b..f1ec5dd581 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -66,11 +66,11 @@ function completenessCheck() { # This is stripped from the output as it is not present in the flattened pom. # Only dependencies with 'compile' or 'runtime' scope are included from original dependency list. msg "Generating dependency list using original pom..." - mvn dependency:list -f pom.xml -DincludeScope=runtime -DexcludeArtifactIds=gson,commons-codec,commons-logging,opencensus-api,opencensus-contrib-http-util,httpclient,httpcore,protobuf-java-util -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e 's/ --.*//' >.org-list.txt + mvn dependency:list -f pom.xml -DincludeScope=runtime -DexcludeArtifactIds=gson,commons-codec,commons-logging,opencensus-api,opencensus-contrib-http-util,httpclient,httpcore,protobuf-java-util,google-http-client -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e 's/ --.*//' >.org-list.txt # Output dep list generated using the flattened pom (only 'compile' and 'runtime' scopes) msg "Generating dependency list using flattened pom..." - mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -DexcludeArtifactIds=gson,commons-codec,commons-logging,opencensus-api,opencensus-contrib-http-util,httpclient,httpcore,protobuf-java-util -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt + mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -DexcludeArtifactIds=gson,commons-codec,commons-logging,opencensus-api,opencensus-contrib-http-util,httpclient,httpcore,protobuf-java-util,google-http-client -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt # Compare two dependency lists msg "Comparing dependency lists..." diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index a3f7fb9d49..e20330c3ca 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17" + value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index 4c7225ec92..0fd6ba2fa0 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/graalvm" + value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.0" } env_vars: { diff --git a/CHANGELOG.md b/CHANGELOG.md index 294ff8aa77..42b452bca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [2.27.0](https://github.com/googleapis/java-bigquerystorage/compare/v2.26.0...v2.27.0) (2022-12-12) + + +### Features + +* Change one thread per retry to use a thread pool ([#1898](https://github.com/googleapis/java-bigquerystorage/issues/1898)) ([44a4e4d](https://github.com/googleapis/java-bigquerystorage/commit/44a4e4d8c70bf910f96d2d2c181d1f23b70f3e52)) +* Throw error when using connection pool for explicit stream ([#1903](https://github.com/googleapis/java-bigquerystorage/issues/1903)) ([bd89556](https://github.com/googleapis/java-bigquerystorage/commit/bd895567fe33735294065d7043d845f14f33f8a8)) + + +### Bug Fixes + +* Add back the accidentally removed background executor provider ([#1899](https://github.com/googleapis/java-bigquerystorage/issues/1899)) ([065cc4f](https://github.com/googleapis/java-bigquerystorage/commit/065cc4f35ad5e5cdc393e01adbea881c6e92dcab)) +* Update JsonStreamWriterBuilder comment and update sample to use the latest schema retrieval support ([#1902](https://github.com/googleapis/java-bigquerystorage/issues/1902)) ([2a46ec3](https://github.com/googleapis/java-bigquerystorage/commit/2a46ec3df419bbeba6a2e60b2b621dc60f31eab1)) + + +### Dependencies + +* Update arrow.version to v10.0.1 ([#1894](https://github.com/googleapis/java-bigquerystorage/issues/1894)) ([8e90767](https://github.com/googleapis/java-bigquerystorage/commit/8e907677bae0c48e8c64be5cbab8fb203645f4ef)) +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.1.0 ([#1905](https://github.com/googleapis/java-bigquerystorage/issues/1905)) ([767be03](https://github.com/googleapis/java-bigquerystorage/commit/767be035defbcc5b77f1276bb1a2983b6a7423bd)) +* Update dependency org.apache.arrow:arrow-memory-netty to v10.0.1 ([#1895](https://github.com/googleapis/java-bigquerystorage/issues/1895)) ([8b079cc](https://github.com/googleapis/java-bigquerystorage/commit/8b079cc47ca914756bd73dda98bd15393754ebb1)) +* Update dependency org.apache.arrow:arrow-vector to v10.0.1 ([#1896](https://github.com/googleapis/java-bigquerystorage/issues/1896)) ([087ac6e](https://github.com/googleapis/java-bigquerystorage/commit/087ac6e7ad813dc248e9029f91c454299a87b2b3)) + ## [2.26.0](https://github.com/googleapis/java-bigquerystorage/compare/v2.25.0...v2.26.0) (2022-11-18) diff --git a/README.md b/README.md index 63f2fdc87c..f86efd230d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.1.4 + 26.1.5 pom import @@ -41,7 +41,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigquerystorage - 2.25.0 + 2.26.0 ``` @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigquerystorage' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquerystorage:2.25.0' +implementation 'com.google.cloud:google-cloud-bigquerystorage:2.26.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.25.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.26.0" ``` ## Authentication diff --git a/google-cloud-bigquerystorage-bom/pom.xml b/google-cloud-bigquerystorage-bom/pom.xml index d219715b0b..0626b05b0a 100644 --- a/google-cloud-bigquerystorage-bom/pom.xml +++ b/google-cloud-bigquerystorage-bom/pom.xml @@ -3,12 +3,12 @@ 4.0.0 com.google.cloud google-cloud-bigquerystorage-bom - 2.26.0 + 2.27.0 pom com.google.cloud google-cloud-shared-config - 1.5.4 + 1.5.5 Google Cloud bigquerystorage BOM @@ -52,37 +52,37 @@ com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.27.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.150.0 + 0.151.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.150.0 + 0.151.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 2.26.0 + 2.27.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.150.0 + 0.151.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.150.0 + 0.151.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 2.26.0 + 2.27.0 diff --git a/google-cloud-bigquerystorage/pom.xml b/google-cloud-bigquerystorage/pom.xml index b3f3e12108..961c55c09d 100644 --- a/google-cloud-bigquerystorage/pom.xml +++ b/google-cloud-bigquerystorage/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.27.0 jar BigQuery Storage https://github.com/googleapis/java-bigquerystorage @@ -11,7 +11,7 @@ com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 google-cloud-bigquerystorage @@ -206,10 +206,6 @@ google-cloud-core runtime - - com.google.protobuf - protobuf-java - com.google.api.grpc diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 6380af4fc6..d2c58e372b 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -320,6 +320,11 @@ private void setStreamWriterSettings( * newBuilder that constructs a JsonStreamWriter builder with BigQuery client being initialized by * StreamWriter by default. * + *

The table schema passed in will be updated automatically when there is a schema update + * event. When used for Writer creation, it should be the latest schema. So when you are trying to + * reuse a stream, you should use Builder newBuilder( String streamOrTableName, + * BigQueryWriteClient client) instead, so the created Writer will be based on a fresh schema. + * * @param streamOrTableName name of the stream that must follow * "projects/[^/]+/datasets/[^/]+/tables/[^/]+/streams/[^/]+" or table name * "projects/[^/]+/datasets/[^/]+/tables/[^/]+" @@ -336,6 +341,11 @@ public static Builder newBuilder(String streamOrTableName, TableSchema tableSche /** * newBuilder that constructs a JsonStreamWriter builder. * + *

The table schema passed in will be updated automatically when there is a schema update + * event. When used for Writer creation, it should be the latest schema. So when you are trying to + * reuse a stream, you should use Builder newBuilder( String streamOrTableName, + * BigQueryWriteClient client) instead, so the created Writer will be based on a fresh schema. + * * @param streamOrTableName name of the stream that must follow * "projects/[^/]+/datasets/[^/]+/tables/[^/]+/streams/[^/]+" * @param tableSchema The schema of the table when the stream was created, which is passed back diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java index f9e6186edc..4d07dfdd91 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java @@ -47,7 +47,10 @@ public class StreamWriter implements AutoCloseable { private static final Logger log = Logger.getLogger(StreamWriter.class.getName()); private static String datasetsMatching = "projects/[^/]+/datasets/[^/]+/"; - private static Pattern streamPattern = Pattern.compile(datasetsMatching); + private static Pattern streamPatternDatasets = Pattern.compile(datasetsMatching); + + private static String defaultStreamMatching = "/_default"; + private static Pattern streamPatternDefaultStream = Pattern.compile(defaultStreamMatching); // Cache of location info for a given dataset. private static Map projectAndDatasetToLocation = new ConcurrentHashMap<>(); @@ -195,6 +198,14 @@ private StreamWriter(Builder builder) throws IOException { getBigQueryWriteClient(builder), ownsBigQueryWriteClient)); } else { + if (!isDefaultStream(streamName)) { + log.warning( + "Connection pool is only allowed in default stream! However received " + + builder.streamName); + throw new IllegalArgumentException( + "Trying to enable connection pool in non-default stream."); + } + BigQueryWriteClient client = getBigQueryWriteClient(builder); String location = builder.location; if (location == null || location.isEmpty()) { @@ -264,7 +275,7 @@ private StreamWriter(Builder builder) throws IOException { @VisibleForTesting static String extractDatasetAndProjectName(String streamName) { - Matcher streamMatcher = streamPattern.matcher(streamName); + Matcher streamMatcher = streamPatternDatasets.matcher(streamName); if (streamMatcher.find()) { return streamMatcher.group(); } else { @@ -273,12 +284,19 @@ static String extractDatasetAndProjectName(String streamName) { } } + @VisibleForTesting + static boolean isDefaultStream(String streamName) { + Matcher streamMatcher = streamPatternDefaultStream.matcher(streamName); + return streamMatcher.find(); + } + private BigQueryWriteClient getBigQueryWriteClient(Builder builder) throws IOException { if (builder.client == null) { BigQueryWriteSettings stubSettings = BigQueryWriteSettings.newBuilder() .setCredentialsProvider(builder.credentialsProvider) .setTransportChannelProvider(builder.channelProvider) + .setBackgroundExecutorProvider(builder.executorProvider) .setEndpoint(builder.endpoint) .build(); testOnlyClientCreatedTimes++; diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index 258a287a1c..8c34ad9b3c 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -63,8 +63,8 @@ @RunWith(JUnit4.class) public class JsonStreamWriterTest { private static final Logger LOG = Logger.getLogger(JsonStreamWriterTest.class.getName()); - private static final String TEST_STREAM = "projects/p/datasets/d/tables/t/streams/s"; - private static final String TEST_STREAM_2 = "projects/p/datasets/d2/tables/t2/streams/s2"; + private static final String TEST_STREAM = "projects/p/datasets/d/tables/t/streams/_default"; + private static final String TEST_STREAM_2 = "projects/p/datasets/d2/tables/t2/streams/_default"; private static final String TEST_TABLE = "projects/p/datasets/d/tables/t"; private static final ExecutorProvider SINGLE_THREAD_EXECUTOR = InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(1).build(); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java index 134b438593..50e43fe45d 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java @@ -62,8 +62,10 @@ @RunWith(JUnit4.class) public class StreamWriterTest { private static final Logger log = Logger.getLogger(StreamWriterTest.class.getName()); - private static final String TEST_STREAM_1 = "projects/p/datasets/d1/tables/t1/streams/s1"; - private static final String TEST_STREAM_2 = "projects/p/datasets/d2/tables/t2/streams/s2"; + private static final String TEST_STREAM_1 = "projects/p/datasets/d1/tables/t1/streams/_default"; + private static final String TEST_STREAM_2 = "projects/p/datasets/d2/tables/t2/streams/_default"; + private static final String TEST_STREAM_SHORTEN = "projects/p/datasets/d2/tables/t2/_default"; + private static final String EXPLICIT_STEAM = "projects/p/datasets/d1/tables/t1/streams/s1"; private static final String TEST_TRACE_ID = "DATAFLOW:job_id"; private FakeScheduledExecutorService fakeExecutor; private FakeBigQueryWrite testBigQueryWrite; @@ -366,6 +368,31 @@ public void run() throws Throwable { }); } + @Test + public void testEnableConnectionPoolOnExplicitStream() throws Exception { + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + new ThrowingRunnable() { + @Override + public void run() throws Throwable { + StreamWriter.newBuilder(EXPLICIT_STEAM, client) + .setEnableConnectionPool(true) + .build(); + } + }); + assertTrue(ex.getMessage().contains("Trying to enable connection pool in non-default stream.")); + } + + @Test + public void testShortenStreamNameAllowed() throws Exception { + // no exception is thrown. + StreamWriter.newBuilder(TEST_STREAM_SHORTEN, client) + .setEnableConnectionPool(true) + .setLocation("us") + .build(); + } + @Test public void testAppendSuccessAndConnectionError() throws Exception { StreamWriter writer = getTestStreamWriter(); diff --git a/grpc-google-cloud-bigquerystorage-v1/pom.xml b/grpc-google-cloud-bigquerystorage-v1/pom.xml index a36c5d5b5d..70ebdd87a9 100644 --- a/grpc-google-cloud-bigquerystorage-v1/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 2.26.0 + 2.27.0 grpc-google-cloud-bigquerystorage-v1 GRPC library for grpc-google-cloud-bigquerystorage-v1 com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 diff --git a/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml b/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml index 778903f97d..bf6472829e 100644 --- a/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1beta1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.150.0 + 0.151.0 grpc-google-cloud-bigquerystorage-v1beta1 GRPC library for grpc-google-cloud-bigquerystorage-v1beta1 com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 diff --git a/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml b/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml index 282b5213fb..08171ba24e 100644 --- a/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml +++ b/grpc-google-cloud-bigquerystorage-v1beta2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.150.0 + 0.151.0 grpc-google-cloud-bigquerystorage-v1beta2 GRPC library for grpc-google-cloud-bigquerystorage-v1beta2 com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 diff --git a/pom.xml b/pom.xml index c1116df39b..f7abd87805 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-bigquerystorage-parent pom - 2.26.0 + 2.27.0 BigQuery Storage Parent https://github.com/googleapis/java-bigquerystorage @@ -14,7 +14,7 @@ com.google.cloud google-cloud-shared-config - 1.5.4 + 1.5.5 @@ -76,44 +76,44 @@ com.google.cloud google-cloud-shared-dependencies - 3.0.6 + 3.1.0 pom import com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.150.0 + 0.151.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.150.0 + 0.151.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 2.26.0 + 2.27.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta1 - 0.150.0 + 0.151.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1beta2 - 0.150.0 + 0.151.0 com.google.api.grpc grpc-google-cloud-bigquerystorage-v1 - 2.26.0 + 2.27.0 com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.27.0 org.json diff --git a/proto-google-cloud-bigquerystorage-v1/pom.xml b/proto-google-cloud-bigquerystorage-v1/pom.xml index 6213582a18..4380700cc8 100644 --- a/proto-google-cloud-bigquerystorage-v1/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1 - 2.26.0 + 2.27.0 proto-google-cloud-bigquerystorage-v1 PROTO library for proto-google-cloud-bigquerystorage-v1 com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 diff --git a/proto-google-cloud-bigquerystorage-v1beta1/pom.xml b/proto-google-cloud-bigquerystorage-v1beta1/pom.xml index 0108942e56..1d9b326e49 100644 --- a/proto-google-cloud-bigquerystorage-v1beta1/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1beta1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta1 - 0.150.0 + 0.151.0 proto-google-cloud-bigquerystorage-v1beta1 PROTO library for proto-google-cloud-bigquerystorage-v1beta1 com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 diff --git a/proto-google-cloud-bigquerystorage-v1beta2/pom.xml b/proto-google-cloud-bigquerystorage-v1beta2/pom.xml index 878866a0c2..ef3e2c3632 100644 --- a/proto-google-cloud-bigquerystorage-v1beta2/pom.xml +++ b/proto-google-cloud-bigquerystorage-v1beta2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigquerystorage-v1beta2 - 0.150.0 + 0.151.0 proto-google-cloud-bigquerystorage-v1beta2 PROTO library for proto-google-cloud-bigquerystorage-v1beta2 com.google.cloud google-cloud-bigquerystorage-parent - 2.26.0 + 2.27.0 diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index cc5a483690..a64e59bfc9 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -21,7 +21,7 @@ 1.8 1.8 UTF-8 - 10.0.0 + 10.0.1 @@ -30,7 +30,7 @@ com.google.cloud google-cloud-bigquerystorage - 2.25.0 + 2.26.0 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 7f6eec822b..9bd6c72b9d 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -21,7 +21,7 @@ 1.8 1.8 UTF-8 - 10.0.0 + 10.0.1 @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigquerystorage - 2.26.0 + 2.27.0 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index b2039fdc1d..a093a969af 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -21,7 +21,7 @@ 1.8 1.8 UTF-8 - 10.0.0 + 10.0.1 @@ -31,7 +31,7 @@ com.google.cloud libraries-bom - 26.1.4 + 26.1.5 pom import diff --git a/samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java b/samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java index a95388b47f..d8f0cc38b5 100644 --- a/samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java +++ b/samples/snippets/src/main/java/com/example/bigquerystorage/WriteToDefaultStream.java @@ -24,16 +24,14 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableResult; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1.Exceptions; import com.google.cloud.bigquery.storage.v1.Exceptions.AppendSerializtionError; import com.google.cloud.bigquery.storage.v1.Exceptions.StorageException; import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; import com.google.cloud.bigquery.storage.v1.TableName; -import com.google.cloud.bigquery.storage.v1.TableSchema; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.Descriptors.DescriptorValidationException; @@ -41,6 +39,8 @@ import io.grpc.Status.Code; import java.io.IOException; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Phaser; import javax.annotation.concurrent.GuardedBy; import org.json.JSONArray; @@ -144,17 +144,12 @@ private static class DataWriter { public void initialize(TableName parentTable) throws DescriptorValidationException, IOException, InterruptedException { - // Retrive table schema information. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Table table = bigquery.getTable(parentTable.getDataset(), parentTable.getTable()); - Schema schema = table.getDefinition().getSchema(); - TableSchema tableSchema = BqToBqStorageSchemaConverter.convertTableSchema(schema); - // Use the JSON stream writer to send records in JSON format. Specify the table name to write // to the default stream. // For more information about JsonStreamWriter, see: // https://googleapis.dev/java/google-cloud-bigquerystorage/latest/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.html - streamWriter = JsonStreamWriter.newBuilder(parentTable.toString(), tableSchema).build(); + streamWriter = + JsonStreamWriter.newBuilder(parentTable.toString(), BigQueryWriteClient.create()).build(); } public void append(AppendContext appendContext) @@ -193,6 +188,8 @@ static class AppendCompleteCallback implements ApiFutureCallback { - try { - // Since default stream appends are not ordered, we can simply retry the - // appends. - // Retrying with exclusive streams requires more careful consideration. - this.parent.append(appendContext); - } catch (Exception e) { - // Fall through to return error. - System.out.format("Failed to retry append: %s%n", e); - } - }) - .start(); + pool.submit( + () -> { + try { + // Since default stream appends are not ordered, we can simply retry the + // appends. + // Retrying with exclusive streams requires more careful consideration. + this.parent.append(appendContext); + } catch (Exception e) { + // Fall through to return error. + System.out.format("Failed to retry append: %s%n", e); + } + }); // Mark the existing attempt as done since it's being retried. done(); return; @@ -251,15 +247,14 @@ public void onFailure(Throwable throwable) { // Retry the remaining valid rows, but using a separate thread to // avoid potentially blocking while we are in a callback. if (dataNew.length() > 0) { - new Thread( - () -> { - try { - this.parent.append(new AppendContext(dataNew, 0)); - } catch (Exception e2) { - System.out.format("Failed to retry append with filtered rows: %s%n", e2); - } - }) - .start(); + pool.submit( + () -> { + try { + this.parent.append(new AppendContext(dataNew, 0)); + } catch (Exception e2) { + System.out.format("Failed to retry append with filtered rows: %s%n", e2); + } + }); } return; } diff --git a/tutorials/JsonWriterDefaultStream/pom.xml b/tutorials/JsonWriterDefaultStream/pom.xml index d25fe380e1..aaeff34348 100644 --- a/tutorials/JsonWriterDefaultStream/pom.xml +++ b/tutorials/JsonWriterDefaultStream/pom.xml @@ -19,7 +19,7 @@ com.google.cloud google-cloud-bigquerystorage - 2.25.0 + 2.26.0 com.google.cloud @@ -34,12 +34,12 @@ org.apache.arrow arrow-vector - 10.0.0 + 10.0.1 org.apache.arrow arrow-memory-netty - 10.0.0 + 10.0.1 diff --git a/versions.txt b/versions.txt index 914f13d9d2..9770333ff6 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-bigquerystorage:2.26.0:2.26.0 -grpc-google-cloud-bigquerystorage-v1beta1:0.150.0:0.150.0 -grpc-google-cloud-bigquerystorage-v1beta2:0.150.0:0.150.0 -grpc-google-cloud-bigquerystorage-v1:2.26.0:2.26.0 -proto-google-cloud-bigquerystorage-v1beta1:0.150.0:0.150.0 -proto-google-cloud-bigquerystorage-v1beta2:0.150.0:0.150.0 -proto-google-cloud-bigquerystorage-v1:2.26.0:2.26.0 +google-cloud-bigquerystorage:2.27.0:2.27.0 +grpc-google-cloud-bigquerystorage-v1beta1:0.151.0:0.151.0 +grpc-google-cloud-bigquerystorage-v1beta2:0.151.0:0.151.0 +grpc-google-cloud-bigquerystorage-v1:2.27.0:2.27.0 +proto-google-cloud-bigquerystorage-v1beta1:0.151.0:0.151.0 +proto-google-cloud-bigquerystorage-v1beta2:0.151.0:0.151.0 +proto-google-cloud-bigquerystorage-v1:2.27.0:2.27.0