From 9ff12b4481a1c0a618c5df413395abf70809f4ea Mon Sep 17 00:00:00 2001 From: Igor Berntein Date: Mon, 11 Dec 2023 12:03:26 -0500 Subject: [PATCH 1/7] test: add integration test to ensure that reverse scan resumption produces correct results Change-Id: I909ec95bebe87219d66b387cd80e7095e75252a2 --- .../cloud/bigtable/data/v2/it/ReadIT.java | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 7b58e14f7c..85487ff1ab 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -18,13 +18,17 @@ import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.TruthJUnit.assume; +import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; import com.google.api.core.SettableApiFuture; +import com.google.api.gax.batching.Batcher; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.StreamController; import com.google.cloud.bigtable.data.v2.BigtableDataClient; +import com.google.cloud.bigtable.data.v2.BigtableDataSettings; import com.google.cloud.bigtable.data.v2.models.BulkMutation; import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange; @@ -38,8 +42,17 @@ import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ManagedChannelBuilder; +import io.grpc.MethodDescriptor; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -308,6 +321,90 @@ public void reversed() { .inOrder(); } + @Test + public void reversedWithFocedResumption() throws IOException, InterruptedException { + assume() + .withMessage("reverse scans are not supported in the emulator") + .that(testEnvRule.env()) + .isNotInstanceOf(EmulatorEnv.class); + + BigtableDataClient client = testEnvRule.env().getDataClient(); + String tableId = testEnvRule.env().getTableId(); + String familyId = testEnvRule.env().getFamilyId(); + String uniqueKey = prefix + "-rev-queries2"; + + // Add enough rows that ensures resumption logic is forced + Random random; + List expectedResults; + try (Batcher batcher = client.newBulkMutationBatcher(tableId)) { + + byte[] valueBytes = new byte[1024]; + random = new Random(); + + expectedResults = new ArrayList<>(); + + for (int i = 0; i < 2 * 1024; i++) { + ByteString key = ByteString.copyFromUtf8(String.format("%s-%05d", uniqueKey, i)); + ByteString qualifier = ByteString.copyFromUtf8("q"); + long timestamp = System.currentTimeMillis() * 1000; + random.nextBytes(valueBytes); + ByteString value = ByteString.copyFrom(valueBytes); + + batcher.add(RowMutationEntry.create(key).setCell(familyId, qualifier, timestamp, value)); + expectedResults.add( + Row.create( + key, + ImmutableList.of( + RowCell.create(familyId, qualifier, timestamp, ImmutableList.of(), value)))); + } + } + Collections.reverse(expectedResults); + + BigtableDataSettings.Builder settingsBuilder = + testEnvRule.env().getDataClientSettings().toBuilder(); + + InstantiatingGrpcChannelProvider.Builder transport = + ((InstantiatingGrpcChannelProvider) + settingsBuilder.stubSettings().getTransportChannelProvider()) + .toBuilder(); + ApiFunction oldConfigurator = + transport.getChannelConfigurator(); + + // Randomly camp the deadline to force a timeout to force a retry + transport.setChannelConfigurator( + (ManagedChannelBuilder c) -> { + if (oldConfigurator != null) { + c = oldConfigurator.apply(c); + } + return c.intercept( + new ClientInterceptor() { + @Override + public ClientCall interceptCall( + MethodDescriptor method, CallOptions callOptions, Channel next) { + if (method.getBareMethodName().equals("ReadRows")) { + callOptions = + callOptions.withDeadlineAfter(random.nextInt(200), TimeUnit.MILLISECONDS); + } + + return next.newCall(method, callOptions); + } + }); + }); + settingsBuilder.stubSettings().setTransportChannelProvider(transport.build()); + + try (BigtableDataClient patchedClient = BigtableDataClient.create(settingsBuilder.build())) { + for (int i = 0; i < 10; i++) { + List actualResults = new ArrayList<>(); + for (Row row : + patchedClient.readRows(Query.create(tableId).prefix(uniqueKey).reversed(true))) { + actualResults.add(row); + } + assertThat(actualResults).containsExactlyElementsIn(expectedResults).inOrder(); + Thread.sleep(10); + } + } + } + @Test public void readSingleNonexistentAsyncCallback() throws Exception { ApiFuture future = From 90f97e76948cffcb1bf28a2b3e48aa325e4960a8 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 11 Dec 2023 17:12:01 +0000 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c7027f42f..d9ba07b218 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.27.0') +implementation platform('com.google.cloud:libraries-bom:26.28.0') implementation 'com.google.cloud:google-cloud-bigtable' ``` From 66516f9ccdf944c09c43bf96c08bfa6590f53642 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 11 Dec 2023 17:12:35 +0000 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c7027f42f..d9ba07b218 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.27.0') +implementation platform('com.google.cloud:libraries-bom:26.28.0') implementation 'com.google.cloud:google-cloud-bigtable' ``` From 39bb429288e9fcc0e4872facef0d2a7a8cb49c45 Mon Sep 17 00:00:00 2001 From: Igor Berntein Date: Mon, 11 Dec 2023 14:34:07 -0500 Subject: [PATCH 4/7] tweaks Change-Id: Ic7027670c7bffdffa569218bb771218c5a689f54 --- .../com/google/cloud/bigtable/data/v2/it/ReadIT.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 85487ff1ab..c608d84005 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -64,6 +64,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.threeten.bp.Duration; @RunWith(JUnit4.class) public class ReadIT { @@ -322,7 +323,7 @@ public void reversed() { } @Test - public void reversedWithFocedResumption() throws IOException, InterruptedException { + public void reversedWithForcedResumption() throws IOException, InterruptedException { assume() .withMessage("reverse scans are not supported in the emulator") .that(testEnvRule.env()) @@ -363,6 +364,10 @@ public void reversedWithFocedResumption() throws IOException, InterruptedExcepti BigtableDataSettings.Builder settingsBuilder = testEnvRule.env().getDataClientSettings().toBuilder(); + settingsBuilder.stubSettings().readRowsSettings().retrySettings() + .setInitialRpcTimeout(Duration.ofMillis(10)) + .setMaxAttempts(100); + InstantiatingGrpcChannelProvider.Builder transport = ((InstantiatingGrpcChannelProvider) settingsBuilder.stubSettings().getTransportChannelProvider()) @@ -398,9 +403,9 @@ public ClientCall interceptCall( for (Row row : patchedClient.readRows(Query.create(tableId).prefix(uniqueKey).reversed(true))) { actualResults.add(row); + Thread.sleep(1); } assertThat(actualResults).containsExactlyElementsIn(expectedResults).inOrder(); - Thread.sleep(10); } } } From 91bcc1810e21305d1047dc458ea901d8ed3fc12d Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 11 Dec 2023 19:37:48 +0000 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../com/google/cloud/bigtable/data/v2/it/ReadIT.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index c608d84005..93ed6ee229 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -364,9 +364,12 @@ public void reversedWithForcedResumption() throws IOException, InterruptedExcept BigtableDataSettings.Builder settingsBuilder = testEnvRule.env().getDataClientSettings().toBuilder(); - settingsBuilder.stubSettings().readRowsSettings().retrySettings() - .setInitialRpcTimeout(Duration.ofMillis(10)) - .setMaxAttempts(100); + settingsBuilder + .stubSettings() + .readRowsSettings() + .retrySettings() + .setInitialRpcTimeout(Duration.ofMillis(10)) + .setMaxAttempts(100); InstantiatingGrpcChannelProvider.Builder transport = ((InstantiatingGrpcChannelProvider) From b89b6980b761eeb8ff222064ffde0427362c11ed Mon Sep 17 00:00:00 2001 From: Igor Berntein Date: Mon, 11 Dec 2023 14:44:54 -0500 Subject: [PATCH 6/7] stale code Change-Id: I00d8020a441a6520d22a91ee12b825a05b840040 --- .../test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 93ed6ee229..61652680bf 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -368,7 +368,6 @@ public void reversedWithForcedResumption() throws IOException, InterruptedExcept .stubSettings() .readRowsSettings() .retrySettings() - .setInitialRpcTimeout(Duration.ofMillis(10)) .setMaxAttempts(100); InstantiatingGrpcChannelProvider.Builder transport = From b13766b85ba4679aa813e27a87def62edbc58760 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 11 Dec 2023 19:48:12 +0000 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../java/com/google/cloud/bigtable/data/v2/it/ReadIT.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java index 61652680bf..0fa7eb10bd 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/ReadIT.java @@ -64,7 +64,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.threeten.bp.Duration; @RunWith(JUnit4.class) public class ReadIT { @@ -364,11 +363,7 @@ public void reversedWithForcedResumption() throws IOException, InterruptedExcept BigtableDataSettings.Builder settingsBuilder = testEnvRule.env().getDataClientSettings().toBuilder(); - settingsBuilder - .stubSettings() - .readRowsSettings() - .retrySettings() - .setMaxAttempts(100); + settingsBuilder.stubSettings().readRowsSettings().retrySettings().setMaxAttempts(100); InstantiatingGrpcChannelProvider.Builder transport = ((InstantiatingGrpcChannelProvider)