From 5f8b6823521b8fefac5b2ee7093181afabc38c72 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 10 Nov 2020 09:34:22 -0800 Subject: [PATCH 01/10] chore(java): ignore return code 28 in README autosynth job (#605) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/268ed0db-90ce-4f9f-ba46-1924fe46218f/targets - [ ] To automatically regenerate this PR, check this box. Source-Link: https://github.com/googleapis/synthtool/commit/7db8a6c5ffb12a6e4c2f799c18f00f7f3d60e279 --- .kokoro/readme.sh | 11 ++++++++++- synth.metadata | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.kokoro/readme.sh b/.kokoro/readme.sh index 65bff26aba..013ce452f6 100755 --- a/.kokoro/readme.sh +++ b/.kokoro/readme.sh @@ -28,9 +28,18 @@ echo "https://${GITHUB_TOKEN}:@github.com" >> ~/.git-credentials git config --global credential.helper 'store --file ~/.git-credentials' python3.6 -m pip install git+https://github.com/googleapis/synthtool.git#egg=gcp-synthtool + +set +e python3.6 -m autosynth.synth \ --repository=googleapis/java-storage \ --synth-file-name=.github/readme/synth.py \ --metadata-path=.github/readme/synth.metadata \ --pr-title="chore: regenerate README" \ - --branch-suffix="readme" \ No newline at end of file + --branch-suffix="readme" + +# autosynth returns 28 to signal there are no changes +RETURN_CODE=$? +if [[ ${RETURN_CODE} -ne 0 && ${RETURN_CODE} -ne 28 ]] +then + exit ${RETURN_CODE} +fi diff --git a/synth.metadata b/synth.metadata index 5e7ee883cf..675b68c047 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "b39a67551866fb8e09755e4918f30f92754ab297" + "sha": "1e399cd33755e647bf08f4a82af932320cab655d" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "c7824ea48ff6d4d42dfae0849aec8a85acd90bd9" + "sha": "7db8a6c5ffb12a6e4c2f799c18f00f7f3d60e279" } } ], From b3e09c0091857f0d4b16aa61fba0088da3e760dd Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 10 Nov 2020 09:52:06 -0800 Subject: [PATCH 02/10] chore: regenerate README (#608) This PR was generated using Autosynth. :rainbow:
Log from Synthtool ``` 2020-11-10 17:37:30,214 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-storage/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working directory clean 2020-11-10 17:37:31,099 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
Full log will be available here: https://source.cloud.google.com/results/invocations/6958dd6b-14e5-4a47-98af-51ee2eb3e34e/targets - [ ] To automatically regenerate this PR, check this box. --- .github/readme/synth.metadata/synth.metadata | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index f8ff395fd8..c66d6f43e1 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "bacc7b75898c3d4054214da069588ad4957b5d39" + "sha": "5f8b6823521b8fefac5b2ee7093181afabc38c72" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2c8aecedd55b0480fb4e123b6e07fa5b12953862" + "sha": "7db8a6c5ffb12a6e4c2f799c18f00f7f3d60e279" } } ] diff --git a/README.md b/README.md index 2f99ca640b..df20b53ccd 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,11 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.cloud:google-cloud-storage:1.113.2' +compile 'com.google.cloud:google-cloud-storage:1.113.3' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.113.2" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "1.113.3" ``` ## Authentication From 4274770f7fd6e59436d32ee87f0e5b685e42a430 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2020 17:52:10 +0000 Subject: [PATCH 03/10] chore: release 1.113.4-SNAPSHOT (#607) :robot: I have created a release \*beep\* \*boop\* --- ### Updating meta-information for bleeding-edge SNAPSHOT release. --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). --- google-cloud-storage/pom.xml | 4 ++-- pom.xml | 2 +- versions.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 279e25c441..8d2a96750a 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 1.113.3 + 1.113.4-SNAPSHOT jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 1.113.3 + 1.113.4-SNAPSHOT google-cloud-storage diff --git a/pom.xml b/pom.xml index 6bd003cf86..6033f81086 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 1.113.3 + 1.113.4-SNAPSHOT Storage Parent https://github.com/googleapis/java-storage diff --git a/versions.txt b/versions.txt index 11ef215259..d2b8fea4e2 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-storage:1.113.3:1.113.3 \ No newline at end of file +google-cloud-storage:1.113.3:1.113.4-SNAPSHOT \ No newline at end of file From 3ac92c97d371756d7abe0c56419092047b4a4c1b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 10 Nov 2020 22:45:05 +0100 Subject: [PATCH 04/10] chore(deps): update dependency com.google.cloud:google-cloud-storage to v1.113.3 (#606) --- samples/install-without-bom/pom.xml | 2 +- samples/snapshot/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index fe8f70f1a8..99bf72f8e9 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-storage - 1.113.2 + 1.113.3 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 81d033eaf2..6a6fdaebfb 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-storage - 1.113.2 + 1.113.3 From 65fb303253f94be86cf937ef5df3e39ecb884252 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 10 Nov 2020 13:54:11 -0800 Subject: [PATCH 05/10] chore: regenerate README (#609) This PR was generated using Autosynth. :rainbow:
Log from Synthtool ``` 2020-11-10 21:47:08,380 synthtool [DEBUG] > Executing /root/.cache/synthtool/java-storage/.github/readme/synth.py. On branch autosynth-readme nothing to commit, working directory clean 2020-11-10 21:47:09,314 synthtool [DEBUG] > Wrote metadata to .github/readme/synth.metadata/synth.metadata. ```
Full log will be available here: https://source.cloud.google.com/results/invocations/9b008544-21cc-4ca0-8270-b4ac738c3d54/targets - [ ] To automatically regenerate this PR, check this box. --- .github/readme/synth.metadata/synth.metadata | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata index c66d6f43e1..4a44631afe 100644 --- a/.github/readme/synth.metadata/synth.metadata +++ b/.github/readme/synth.metadata/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "5f8b6823521b8fefac5b2ee7093181afabc38c72" + "sha": "3ac92c97d371756d7abe0c56419092047b4a4c1b" } }, { diff --git a/README.md b/README.md index df20b53ccd..4b797d17f6 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-storage - 1.113.2 + 1.113.3 ``` From ac65e5b0bd324d5726504bb3405c758675a56ddc Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 11 Nov 2020 16:58:30 +0100 Subject: [PATCH 06/10] deps: update dependency com.google.cloud:google-cloud-shared-dependencies to v0.15.0 (#610) This PR contains the following updates: | Package | Update | Change | |---|---|---| | [com.google.cloud:google-cloud-shared-dependencies](https://togithub.com/googleapis/java-shared-dependencies) | minor | `0.14.1` -> `0.15.0` | --- ### Release Notes
googleapis/java-shared-dependencies ### [`v0.15.0`](https://togithub.com/googleapis/java-shared-dependencies/blob/master/CHANGELOG.md#​0150-httpswwwgithubcomgoogleapisjava-shared-dependenciescompare0141v0150-2020-11-10) [Compare Source](https://togithub.com/googleapis/java-shared-dependencies/compare/v0.14.1...v0.15.0) ##### Dependencies - update dependency com.google.http-client:google-http-client-bom to v1.38.0 ([#​207](https://www.github.com/googleapis/java-shared-dependencies/issues/207)) ([5e3198d](https://www.github.com/googleapis/java-shared-dependencies/commit/5e3198de5e1d0fe55c4d947b12984db7726059da)) - update google.common-protos.version to v2.0.1 ([#​205](https://www.github.com/googleapis/java-shared-dependencies/issues/205)) ([6b12417](https://www.github.com/googleapis/java-shared-dependencies/commit/6b1241737d34e99a55acdcd439aa71ad28d1e26c)) - update iam.version to v1.0.3 ([#​206](https://www.github.com/googleapis/java-shared-dependencies/issues/206)) ([94a4e31](https://www.github.com/googleapis/java-shared-dependencies/commit/94a4e31634718a206ef0f5ff1a8801dc7be4c5c5)) ##### [0.14.1](https://www.github.com/googleapis/java-shared-dependencies/compare/0.14.0...v0.14.1) (2020-10-31) ##### Dependencies - update google.core.version to v1.93.10 ([#​200](https://www.github.com/googleapis/java-shared-dependencies/issues/200)) ([2d73dc1](https://www.github.com/googleapis/java-shared-dependencies/commit/2d73dc152469d175c74f85c5c76dcc67993c9783))
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/java-storage). --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6033f81086..a09b4b137e 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ UTF-8 github google-cloud-storage-parent - 0.14.1 + 0.15.0 From 48e404fedd5450a63b8ea25017acfaac4c6d5849 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 13 Nov 2020 19:25:15 +0100 Subject: [PATCH 07/10] chore(deps): update dependency com.google.cloud:libraries-bom to v15.1.0 (#615) --- samples/snippets/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 9c0f70a262..7910681e22 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 15.0.0 + 15.1.0 pom import From 216b52c54d34eaf1307788809a3512c461adf381 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Fri, 13 Nov 2020 10:44:10 -0800 Subject: [PATCH 08/10] fix: retry using remote offset (#604) * fix: retry using remote offset * fix: retry using remote offset Co-authored-by: Jesse Lovelace --- .../clirr-ignored-differences.xml | 5 + .../cloud/storage/BlobWriteChannel.java | 113 +++++++++++++++++- .../cloud/storage/spi/v1/HttpStorageRpc.java | 45 +++++++ .../cloud/storage/spi/v1/StorageRpc.java | 9 ++ .../storage/testing/StorageRpcTestBase.java | 5 + .../cloud/storage/BlobWriteChannelTest.java | 68 +++++++++++ 6 files changed, 240 insertions(+), 5 deletions(-) diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml index ba97a975ce..02722fac60 100644 --- a/google-cloud-storage/clirr-ignored-differences.xml +++ b/google-cloud-storage/clirr-ignored-differences.xml @@ -26,4 +26,9 @@ com.google.cloud.storage.BucketInfo$Builder setUpdateTime(java.lang.Long) 7013 + + com/google/cloud/storage/spi/v1/StorageRpc + long getCurrentUploadOffset(java.lang.String) + 7012 + diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java index 0c9520849b..2e9c0a3804 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobWriteChannel.java @@ -51,6 +51,15 @@ class BlobWriteChannel extends BaseWriteChannel { // Contains metadata of the updated object or null if upload is not completed. private StorageObject storageObject; + // Detect if flushBuffer() is being retried or not. + // TODO: I don't think this is thread safe, and there's probably a better way to detect a retry + // occuring. + private boolean retrying = false; + + boolean isRetrying() { + return retrying; + } + StorageObject getStorageObject() { return storageObject; } @@ -63,11 +72,105 @@ protected void flushBuffer(final int length, final boolean last) { new Runnable() { @Override public void run() { - storageObject = - getOptions() - .getStorageRpcV1() - .writeWithResponse( - getUploadId(), getBuffer(), 0, getPosition(), length, last); + if (!isRetrying()) { + // Enable isRetrying state to reduce number of calls to getCurrentUploadOffset() + retrying = true; + storageObject = + getOptions() + .getStorageRpcV1() + .writeWithResponse( + getUploadId(), getBuffer(), 0, getPosition(), length, last); + } else { + // Retriable interruption occurred. + // Variables: + // chunk = getBuffer() + // localNextByteOffset == getPosition() + // chunkSize = getChunkSize() + // + // Case 1: localNextByteOffset == 0 && remoteNextByteOffset == 0: + // we are retrying from first chunk start from 0 offset. + // + // Case 2: localNextByteOffset == remoteNextByteOffset: + // Special case of Case 1 when a chunk is retried. + // + // Case 3: localNextByteOffset < remoteNextByteOffset + // && driftOffset < chunkSize: + // Upload progressed and localNextByteOffset is not in-sync with + // remoteNextByteOffset and driftOffset is less than chunkSize. + // driftOffset must be less than chunkSize for it to retry using + // chunk maintained in memory. + // Find the driftOffset by subtracting localNextByteOffset from + // remoteNextByteOffset. + // Use driftOffset to determine where to restart from using the chunk in + // memory. + // + // Case 4: localNextByteOffset < remoteNextByteOffset + // && driftOffset == chunkSize: + // Special case of Case 3. + // If chunkSize is equal to driftOffset then remoteNextByteOffset has moved on + // to the next chunk. + // + // Case 5: localNextByteOffset < remoteNextByteOffset + // && driftOffset > chunkSize: + // Throw exception as remoteNextByteOffset has drifted beyond the retriable + // chunk maintained in memory. This is not possible unless there's multiple + // clients uploading to the same resumable upload session. + // + // Case 6: localNextByteOffset > remoteNextByteOffset: + // For completeness, this case is not possible because it would require retrying + // a 400 status code which is not allowed. + // + // Get remote offset from API + long remoteNextByteOffset = + getOptions().getStorageRpcV1().getCurrentUploadOffset(getUploadId()); + long localNextByteOffset = getPosition(); + int driftOffset = (int) (remoteNextByteOffset - localNextByteOffset); + int retryChunkLength = length - driftOffset; + + if (localNextByteOffset == 0 && remoteNextByteOffset == 0 + || localNextByteOffset == remoteNextByteOffset) { + // Case 1 and 2 + storageObject = + getOptions() + .getStorageRpcV1() + .writeWithResponse( + getUploadId(), getBuffer(), 0, getPosition(), length, last); + } else if (localNextByteOffset < remoteNextByteOffset + && driftOffset < getChunkSize()) { + // Case 3 + storageObject = + getOptions() + .getStorageRpcV1() + .writeWithResponse( + getUploadId(), + getBuffer(), + driftOffset, + remoteNextByteOffset, + retryChunkLength, + last); + } else if (localNextByteOffset < remoteNextByteOffset + && driftOffset == getChunkSize()) { + // Case 4 + // Continue to next chunk + retrying = false; + return; + } else { + // Case 5 + StringBuilder sb = new StringBuilder(); + sb.append( + "Remote offset has progressed beyond starting byte offset of next chunk."); + sb.append( + "This may be a symptom of multiple clients uploading to the same upload session.\n\n"); + sb.append("For debugging purposes:\n"); + sb.append("uploadId: ").append(getUploadId()).append('\n'); + sb.append("localNextByteOffset: ").append(localNextByteOffset).append('\n'); + sb.append("remoteNextByteOffset: ").append(remoteNextByteOffset).append('\n'); + sb.append("driftOffset: ").append(driftOffset).append("\n\n"); + throw new StorageException(0, sb.toString()); + } + } + // Request was successful and retrying state is now disabled. + retrying = false; } }), getOptions().getRetrySettings(), diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 0960f91ffb..faba82b1de 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -747,6 +747,51 @@ public void write( writeWithResponse(uploadId, toWrite, toWriteOffset, destOffset, length, last); } + @Override + public long getCurrentUploadOffset(String uploadId) { + try { + GenericUrl url = new GenericUrl(uploadId); + HttpRequest httpRequest = storage.getRequestFactory().buildPutRequest(url, null); + + httpRequest.getHeaders().setContentRange("bytes */*"); + // Turn off automatic redirects. + // HTTP 308 are returned if upload is incomplete. + // See: https://cloud.google.com/storage/docs/performing-resumable-uploads + httpRequest.setFollowRedirects(false); + + HttpResponse response = null; + try { + response = httpRequest.execute(); + int code = response.getStatusCode(); + String message = response.getStatusMessage(); + if (code == 201 || code == 200) { + throw new StorageException(0, "Resumable upload is already complete."); + } + StringBuilder sb = new StringBuilder(); + sb.append("Not sure what occurred. Here's debugging information:\n"); + sb.append("Response:\n").append(response.toString()).append("\n\n"); + throw new StorageException(0, sb.toString()); + } catch (HttpResponseException ex) { + int code = ex.getStatusCode(); + if (code == 308 && ex.getHeaders().getRange() == null) { + // No progress has been made. + return 0; + } else { + // API returns last byte received offset + String range = ex.getHeaders().getRange(); + // Return next byte offset by adding 1 to last byte received offset + return Long.parseLong(range.substring(range.indexOf("-") + 1)) + 1; + } + } finally { + if (response != null) { + response.disconnect(); + } + } + } catch (IOException ex) { + throw translate(ex); + } + } + @Override public StorageObject writeWithResponse( String uploadId, diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java index 2088c15bec..cc4dd5740c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java @@ -328,6 +328,15 @@ void write( int length, boolean last); + /** + * Requests current byte offset from Cloud Storage API. Used to recover from a failure in some + * bytes were committed successfully to the open resumable session. + * + * @param uploadId resumable upload ID URL + * @throws StorageException upon failure + */ + long getCurrentUploadOffset(String uploadId); + /** * Writes the provided bytes to a storage object at the provided location. If {@code last=true} * returns metadata of the updated object, otherwise returns null. diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java index 7733f13eb6..b4479682a5 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/testing/StorageRpcTestBase.java @@ -139,6 +139,11 @@ public void write( throw new UnsupportedOperationException("Not implemented yet"); } + @Override + public long getCurrentUploadOffset(String uploadId) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override public StorageObject writeWithResponse( String uploadId, diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java index a18345be89..b6d7bbf3a3 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobWriteChannelTest.java @@ -134,6 +134,74 @@ public void testWriteWithoutFlush() throws IOException { assertEquals(MIN_CHUNK_SIZE, writer.write(ByteBuffer.allocate(MIN_CHUNK_SIZE))); } + @Test + public void testWriteWithFlushRetryChunk() throws IOException { + StorageException exception = new StorageException(new SocketException("Socket closed")); + ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); + Capture capturedBuffer = Capture.newInstance(); + expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(false))) + .andThrow(exception); + expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(0L); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(false))) + .andReturn(null); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer.setChunkSize(MIN_CHUNK_SIZE); + assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); + assertTrue(writer.isOpen()); + assertNull(writer.getStorageObject()); + assertArrayEquals(buffer.array(), capturedBuffer.getValue()); + } + + @Test + public void testWriteWithFlushRetryChunkWithDrift() throws IOException { + StorageException exception = new StorageException(new SocketException("Socket closed")); + ByteBuffer buffer = randomBuffer(MIN_CHUNK_SIZE); + Capture capturedBuffer = Capture.newInstance(); + expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(0), + eq(0L), + eq(MIN_CHUNK_SIZE), + eq(false))) + .andThrow(exception); + expect(storageRpcMock.getCurrentUploadOffset(eq(UPLOAD_ID))).andReturn(10L); + expect( + storageRpcMock.writeWithResponse( + eq(UPLOAD_ID), + capture(capturedBuffer), + eq(10), + eq(10L), + eq(MIN_CHUNK_SIZE - 10), + eq(false))) + .andReturn(null); + replay(storageRpcMock); + writer = new BlobWriteChannel(options, BLOB_INFO, EMPTY_RPC_OPTIONS); + writer.setChunkSize(MIN_CHUNK_SIZE); + assertEquals(MIN_CHUNK_SIZE, writer.write(buffer)); + assertTrue(writer.isOpen()); + assertNull(writer.getStorageObject()); + assertArrayEquals(buffer.array(), capturedBuffer.getValue()); + } + @Test public void testWriteWithFlush() throws IOException { expect(storageRpcMock.open(BLOB_INFO.toPb(), EMPTY_RPC_OPTIONS)).andReturn(UPLOAD_ID); From fea916095ddbe493ba93442884e0ca7e50910783 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Fri, 13 Nov 2020 13:10:07 -0800 Subject: [PATCH 09/10] build(java): use code-suggester v1 tag rather than full semver (#617) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/62d29775-e679-4d20-bb82-8ed0902c03b5/targets - [ ] To automatically regenerate this PR, check this box. Source-Link: https://github.com/googleapis/synthtool/commit/7d652819519dfa24da9e14548232e4aaba71a11c --- .github/workflows/formatting.yaml | 2 +- synth.metadata | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/formatting.yaml b/.github/workflows/formatting.yaml index d4d367cfce..6844407b4d 100644 --- a/.github/workflows/formatting.yaml +++ b/.github/workflows/formatting.yaml @@ -18,7 +18,7 @@ jobs: with: java-version: 11 - run: "mvn com.coveo:fmt-maven-plugin:format" - - uses: googleapis/code-suggester@v1.8.0 + - uses: googleapis/code-suggester@v1 with: command: review pull_number: ${{ github.event.pull_request.number }} diff --git a/synth.metadata b/synth.metadata index 675b68c047..4acdcb8395 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-storage.git", - "sha": "1e399cd33755e647bf08f4a82af932320cab655d" + "sha": "ac65e5b0bd324d5726504bb3405c758675a56ddc" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "7db8a6c5ffb12a6e4c2f799c18f00f7f3d60e279" + "sha": "7d652819519dfa24da9e14548232e4aaba71a11c" } } ], From a61d7707f548f010adba45359d764233eca860f8 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 16 Nov 2020 13:18:48 -0800 Subject: [PATCH 10/10] chore: release 1.113.4 (#620) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 ++++++++++++ google-cloud-storage/pom.xml | 4 ++-- pom.xml | 2 +- versions.txt | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7bb999d38..0ccaff6c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +### [1.113.4](https://www.github.com/googleapis/java-storage/compare/v1.113.3...v1.113.4) (2020-11-13) + + +### Bug Fixes + +* retry using remote offset ([#604](https://www.github.com/googleapis/java-storage/issues/604)) ([216b52c](https://www.github.com/googleapis/java-storage/commit/216b52c54d34eaf1307788809a3512c461adf381)) + + +### Dependencies + +* update dependency com.google.cloud:google-cloud-shared-dependencies to v0.15.0 ([#610](https://www.github.com/googleapis/java-storage/issues/610)) ([ac65e5b](https://www.github.com/googleapis/java-storage/commit/ac65e5b0bd324d5726504bb3405c758675a56ddc)) + ### [1.113.3](https://www.github.com/googleapis/java-storage/compare/v1.113.2...v1.113.3) (2020-11-06) diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 8d2a96750a..32e8e9c8ba 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 1.113.4-SNAPSHOT + 1.113.4 jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,7 +12,7 @@ com.google.cloud google-cloud-storage-parent - 1.113.4-SNAPSHOT + 1.113.4 google-cloud-storage diff --git a/pom.xml b/pom.xml index a09b4b137e..b52e86548b 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 1.113.4-SNAPSHOT + 1.113.4 Storage Parent https://github.com/googleapis/java-storage diff --git a/versions.txt b/versions.txt index d2b8fea4e2..697d9c4621 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -google-cloud-storage:1.113.3:1.113.4-SNAPSHOT \ No newline at end of file +google-cloud-storage:1.113.4:1.113.4 \ No newline at end of file