Skip to content

feat: adding utility to transform protobuf into model object #299

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rahulKQL merged 5 commits into from
May 14, 2020
Merged

feat: adding utility to transform protobuf into model object #299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ea099c4
feat: adding utility to transform protobuf into model object
rahulKQL May 11, 2020
154130d
chore: addressed feedback comments
rahulKQL May 12, 2020
511f924
chore: addressed feedback comments
rahulKQL May 12, 2020
e079eff
chore: removed VisibleForTesting annotation
rahulKQL May 13, 2020
1e680e8
chore: extended unit tests to verify project and instance override
rahulKQL May 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ...e-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/BulkMutation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.MutateRowsRequest;
import com.google.cloud.bigtable.data.v2.internal.NameUtil;
Expand Down Expand Up @@ -114,6 +115,26 @@ public MutateRowsRequest toProto(RequestContext requestContext) {
.build();
}

/**
* Wraps the protobuf {@link MutateRowsRequest}.
*
* <p>This is meant for advanced usage only. Please ensure that the MutateRowsRequest does not use
* server side timestamps. The BigtableDataClient assumes that mutation present in BulkMutation
* are idempotent and is configured to enable retries by default. If serverside timestamps are
* enabled then that can lead to duplicate mutations.
*
* <p>WARNING: when applied, the resulting mutation object will ignore the project id and instance
* id in the table_name and instead apply the configuration in the client.
*/
@BetaApi
public static BulkMutation fromProto(@Nonnull MutateRowsRequest request) {
BulkMutation bulkMutation =
BulkMutation.create(NameUtil.extractTableIdFromTableName(request.getTableName()));
bulkMutation.builder = request.toBuilder();

return bulkMutation;
}

/** Creates a copy of {@link BulkMutation}. */
@Override
public BulkMutation clone() {
Expand Down
17 changes: 17 additions & 0 deletions ...gtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.CheckAndMutateRowRequest;
import com.google.cloud.bigtable.data.v2.internal.NameUtil;
Expand Down Expand Up @@ -136,4 +137,20 @@ public CheckAndMutateRowRequest toProto(RequestContext requestContext) {
.setAppProfileId(requestContext.getAppProfileId())
.build();
}

/**
* Wraps the protobuf {@link CheckAndMutateRowRequest}.
*
* <p>WARNING: Please note that the table_name will be overwritten by the configuration in the
* BigtableDataClient.
*/
@BetaApi
public static ConditionalRowMutation fromProto(@Nonnull CheckAndMutateRowRequest request) {
String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());
ConditionalRowMutation rowMutation =
ConditionalRowMutation.create(tableId, request.getRowKey());
rowMutation.builder = request.toBuilder();

return rowMutation;
}
}
16 changes: 16 additions & 0 deletions google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/Mutation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,22 @@ public static Mutation fromProtoUnsafe(List<com.google.bigtable.v2.Mutation> pro
return mutation;
}

/**
* Constructs a row mutation from an existing protobuf object.
*
* <p>Callers must ensure that the protobuf argument is not using serverside timestamps. The
* client assumes that all mutations are idempotent and will retry in case of transient errors.
* This can lead to row duplication.
*
* <p>When applied, the resulting Mutation object will ignore the project id and instance id in
* the table_name and instead apply the configuration in the client
*/
static Mutation fromProto(List<com.google.bigtable.v2.Mutation> protos) {
Mutation mutation = new Mutation(false);
mutation.mutations.addAll(protos);
return mutation;
}

private Mutation(boolean allowServersideTimestamp) {
this.allowServersideTimestamp = allowServersideTimestamp;
}
Expand Down
17 changes: 17 additions & 0 deletions ...d-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.ReadModifyWriteRowRequest;
import com.google.bigtable.v2.ReadModifyWriteRule;
Expand Down Expand Up @@ -137,4 +138,20 @@ public ReadModifyWriteRowRequest toProto(RequestContext requestContext) {
.setAppProfileId(requestContext.getAppProfileId())
.build();
}

/**
* Wraps the protobuf {@link ReadModifyWriteRowRequest}.
*
* <p>WARNING: Please note that the table_name will be overwritten by the configuration in the
* BigtableDataClient.
*/
@BetaApi
public static ReadModifyWriteRow fromProto(@Nonnull ReadModifyWriteRowRequest request) {
String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());

ReadModifyWriteRow row = ReadModifyWriteRow.create(tableId, request.getRowKey());
row.builder = request.toBuilder();

return row;
}
}
20 changes: 20 additions & 0 deletions ...le-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/RowMutation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.MutateRowRequest;
import com.google.bigtable.v2.MutateRowsRequest;
Expand Down Expand Up @@ -214,4 +215,23 @@ public MutateRowsRequest toBulkProto(RequestContext requestContext) {
Entry.newBuilder().setRowKey(key).addAllMutations(mutation.getMutations()).build())
.build();
}

/**
* Wraps the protobuf {@link MutateRowRequest}.
*
* <p>This is meant for advanced usage only. Please ensure that the MutateRowRequest does not use
* server side timestamps. The BigtableDataClient assumes that RowMutations are idempotent and is
* configured to enable retries by default. If serverside timestamps are enabled, this can lead to
* duplicate mutations.
*
* <p>WARNING: when applied, the resulting mutation object will ignore the project id and instance
* id in the table_name and instead apply the configuration in the client.
*/
@BetaApi
public static RowMutation fromProto(@Nonnull MutateRowRequest request) {
String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());

return RowMutation.create(
tableId, request.getRowKey(), Mutation.fromProto(request.getMutationsList()));
}
}
25 changes: 25 additions & 0 deletions ...oud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/BulkMutationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,4 +147,29 @@ public void addRowMutationEntry() {
bulkMutation.add(entry);
assertThat(bulkMutation.toProto(REQUEST_CONTEXT).getEntriesList()).contains(entry.toProto());
}

@Test
public void fromProtoTest() {
BulkMutation expected =
BulkMutation.create(TABLE_ID)
.add(
"key",
Mutation.create().setCell("fake-family", "fake-qualifier", 10_000L, "fake-value"));

MutateRowsRequest protoRequest = expected.toProto(REQUEST_CONTEXT);
BulkMutation actualBulkMutation = BulkMutation.fromProto(protoRequest);

assertThat(actualBulkMutation.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
MutateRowsRequest overriddenRequest =
actualBulkMutation.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}
25 changes: 25 additions & 0 deletions ...le/src/test/java/com/google/cloud/bigtable/data/v2/models/ConditionalRowMutationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,4 +162,29 @@ public void serializationTest() throws IOException, ClassNotFoundException {
ConditionalRowMutation actual = (ConditionalRowMutation) ois.readObject();
assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT));
}

@Test
public void fromProtoTest() {
ConditionalRowMutation mutation =
ConditionalRowMutation.create(TABLE_ID, TEST_KEY)
.condition(Filters.FILTERS.key().regex("test"))
.then(Mutation.create().setCell("family1", "qualifier1", 10_000L, "value"))
.otherwise(Mutation.create().deleteFamily("family"));

CheckAndMutateRowRequest protoRequest = mutation.toProto(REQUEST_CONTEXT);
ConditionalRowMutation actualRequest = ConditionalRowMutation.fromProto(protoRequest);

assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
CheckAndMutateRowRequest overriddenRequest =
actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}
16 changes: 16 additions & 0 deletions ...e-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/MutationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,4 +254,20 @@ public void testWithLongValue() {
.setValue(ByteString.copyFrom(Longs.toByteArray(20_000L)))
.build());
}

@Test
public void fromProtoTest() {
mutation
.setCell(
"fake-family",
ByteString.copyFromUtf8("fake-qualifier"),
1_000,
ByteString.copyFromUtf8("fake-value"))
.deleteCells("fake-family", ByteString.copyFromUtf8("fake-qualifier"))
.deleteFamily("fake-family2");

List<com.google.bigtable.v2.Mutation> protoMutation = mutation.getMutations();

assertThat(Mutation.fromProto(protoMutation).getMutations()).isEqualTo(protoMutation);
}
}
24 changes: 24 additions & 0 deletions ...gtable/src/test/java/com/google/cloud/bigtable/data/v2/models/ReadModifyWriteRowTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,4 +113,28 @@ public void serializationTest() throws IOException, ClassNotFoundException {
ReadModifyWriteRow actual = (ReadModifyWriteRow) ois.readObject();
assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT));
}

@Test
public void fromProtoTest() {
ReadModifyWriteRow expected =
ReadModifyWriteRow.create(TABLE_ID, "row-key")
.increment("fake-family", ByteString.copyFromUtf8("fake-qualifier"), 1)
.append("fake-family", "fake-qualifier", "fake-value");

ReadModifyWriteRowRequest protoRequest = expected.toProto(REQUEST_CONTEXT);
ReadModifyWriteRow actualRequest = ReadModifyWriteRow.fromProto(protoRequest);

assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
ReadModifyWriteRowRequest overriddenRequest =
actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}
24 changes: 24 additions & 0 deletions ...loud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/models/RowMutationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,4 +139,28 @@ public void testWithLongValue() {
.setValue(ByteString.copyFrom(Longs.toByteArray(100_000L)))
.build());
}

@Test
public void fromProtoTest() {
RowMutation rowMutation =
RowMutation.create("fake-table", "fake-key")
.setCell("fake-family", "fake-qualifier-1", "fake-value")
.setCell("fake-family", "fake-qualifier-2", 30_000L, "fake-value-2");

MutateRowRequest protoRequest = rowMutation.toProto(REQUEST_CONTEXT);
RowMutation actualRequest = RowMutation.fromProto(protoRequest);

assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
MutateRowRequest overriddenRequest =
actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}