diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index 94c91fb72a..53ebca2919 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -105,6 +105,7 @@ import com.google.cloud.bigtable.data.v2.stub.changestream.GenerateInitialChangeStreamPartitionsUserCallable; import com.google.cloud.bigtable.data.v2.stub.changestream.ReadChangeStreamResumptionStrategy; import com.google.cloud.bigtable.data.v2.stub.changestream.ReadChangeStreamUserCallable; +import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracer; import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory; @@ -545,7 +546,12 @@ public ServerStreamingCallable createReadRowsCallable( new TracedServerStreamingCallable<>( readRowsUserCallable, clientContext.getTracerFactory(), span); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.readRowsSettings().getRetrySettings().getTotalTimeout())); } /** @@ -579,7 +585,12 @@ public UnaryCallable createReadRowCallable(RowAdapter UnaryCallable traced = new TracedUnaryCallable<>( firstRow, clientContext.getTracerFactory(), getSpanName("ReadRow")); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.readRowSettings().getRetrySettings().getTotalTimeout())); } else { ServerStreamingCallable readRowsCallable = createReadRowsBaseCallable( @@ -599,7 +610,11 @@ public UnaryCallable createReadRowCallable(RowAdapter return new BigtableUnaryOperationCallable<>( readRowCallable, - clientContext.getDefaultCallContext(), + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.readRowSettings().getRetrySettings().getTotalTimeout()), clientContext.getTracerFactory(), getSpanName("ReadRow"), /*allowNoResponses=*/ true); @@ -715,7 +730,12 @@ private UnaryCallable> createBulkReadRowsCallable( UnaryCallable> traced = new TracedUnaryCallable<>(tracedBatcher, clientContext.getTracerFactory(), span); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.bulkReadRowsSettings().getRetrySettings().getTotalTimeout())); } /** @@ -780,7 +800,14 @@ public ApiFuture> futureCall(String s, ApiCallContext apiCallCon retryable = withRetries(withBigtableTracer, settings.sampleRowKeysSettings()); return createUserFacingUnaryCallable( - methodName, new SampleRowKeysCallableWithRequest(retryable, requestContext)); + methodName, + new SampleRowKeysCallableWithRequest(retryable, requestContext) + .withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.sampleRowKeysSettings().getRetrySettings().getTotalTimeout()))); } /** @@ -903,7 +930,12 @@ private UnaryCallable createMutateRowsBas new TracedUnaryCallable<>( tracedBatcherUnaryCallable, clientContext.getTracerFactory(), spanName); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.bulkMutateRowsSettings().getRetrySettings().getTotalTimeout())); } /** @@ -1108,7 +1140,15 @@ private UnaryCallable createReadModifyWriteRowCallable( ServerStreamingCallable traced = new TracedServerStreamingCallable<>(retrying, clientContext.getTracerFactory(), span); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings + .generateInitialChangeStreamPartitionsSettings() + .getRetrySettings() + .getTotalTimeout())); } /** @@ -1180,7 +1220,12 @@ private UnaryCallable createReadModifyWriteRowCallable( new TracedServerStreamingCallable<>( readChangeStreamUserCallable, clientContext.getTracerFactory(), span); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.readChangeStreamSettings().getRetrySettings().getTotalTimeout())); } /** @@ -1266,7 +1311,13 @@ public Map extract(ExecuteQueryRequest executeQueryRequest) { new TracedServerStreamingCallable<>(retries, clientContext.getTracerFactory(), span); return new ExecuteQueryCallable( - traced.withDefaultCallContext(clientContext.getDefaultCallContext()), requestContext); + traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.executeQuerySettings().getRetrySettings().getTotalTimeout())), + requestContext); } /** @@ -1344,7 +1395,12 @@ public ApiFuture futureCall(ReqT reqT, ApiCallContext apiCallContext) { clientContext.getTracerFactory(), getSpanName(methodDescriptor.getBareMethodName())); - return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); + return traced.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + callSettings.getRetrySettings().getTotalTimeout())); } private UnaryCallable createUnaryCallableNew( @@ -1373,7 +1429,11 @@ private UnaryCallable createUnar return new BigtableUnaryOperationCallable<>( transformed, - clientContext.getDefaultCallContext(), + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + callSettings.getRetrySettings().getTotalTimeout()), clientContext.getTracerFactory(), getSpanName(methodDescriptor.getBareMethodName()), /* allowNoResponse= */ false); @@ -1407,7 +1467,12 @@ public Map extract(PingAndWarmRequest request) { }) .build(), Collections.emptySet()); - return pingAndWarm.withDefaultCallContext(clientContext.getDefaultCallContext()); + return pingAndWarm.withDefaultCallContext( + clientContext + .getDefaultCallContext() + .withOption( + BigtableTracer.OPERATION_TIMEOUT_KEY, + settings.pingAndWarmSettings().getRetrySettings().getTotalTimeout())); } private UnaryCallable withRetries( diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java index fd54313e8d..8aa53fa198 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java @@ -23,6 +23,7 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PER_CONNECTION_ERROR_COUNT_NAME; +import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME; @@ -115,7 +116,8 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter { CLIENT_BLOCKING_LATENCIES_NAME, APPLICATION_BLOCKING_LATENCIES_NAME, RETRY_COUNT_NAME, - CONNECTIVITY_ERROR_COUNT_NAME) + CONNECTIVITY_ERROR_COUNT_NAME, + REMAINING_DEADLINE_NAME) .stream() .map(m -> METER_NAME + m) .collect(ImmutableList.toImmutableList()); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java index d0e307d510..ad77b207b3 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java @@ -16,10 +16,12 @@ package com.google.cloud.bigtable.data.v2.stub.metrics; import com.google.api.core.BetaApi; +import com.google.api.core.InternalApi; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.tracing.ApiTracer; import com.google.api.gax.tracing.BaseApiTracer; import javax.annotation.Nullable; +import org.threeten.bp.Duration; /** * A Bigtable specific {@link ApiTracer} that includes additional contexts. This class is a base @@ -30,6 +32,10 @@ public class BigtableTracer extends BaseApiTracer { private volatile int attempt = 0; + @InternalApi("for internal use only") + public static final ApiCallContext.Key OPERATION_TIMEOUT_KEY = + ApiCallContext.Key.create("OPERATION_TIMEOUT"); + @Override public void attemptStarted(int attemptNumber) { this.attempt = attemptNumber; @@ -93,4 +99,12 @@ public void grpcChannelQueuedLatencies(long queuedTimeMs) { public void grpcMessageSent() { // noop } + + /** + * Record the operation timeout from user settings for calculating remaining deadline. This will + * be called in BuiltinMetricsTracer. + */ + public void setOperationTimeout(Duration operationTimeout) { + // noop + } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index 167cd0dc2e..b977a0a2c7 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -16,6 +16,7 @@ package com.google.cloud.bigtable.data.v2.stub.metrics; import com.google.api.core.InternalApi; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcResponseMetadata; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ResponseObserver; @@ -26,6 +27,7 @@ import com.google.common.base.Stopwatch; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; +import org.threeten.bp.Duration; /** * This callable will @@ -62,6 +64,11 @@ public void call( BigtableTracerResponseObserver innerObserver = new BigtableTracerResponseObserver<>( responseObserver, (BigtableTracer) context.getTracer(), responseMetadata); + GrpcCallContext callContext = (GrpcCallContext) context; + Duration deadline = callContext.getOption(BigtableTracer.OPERATION_TIMEOUT_KEY); + if (deadline != null) { + ((BigtableTracer) context.getTracer()).setOperationTimeout(deadline); + } innerCallable.call( request, innerObserver, diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index 7dfca8b753..1f000c4639 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -19,12 +19,14 @@ import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; import com.google.api.core.InternalApi; +import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcResponseMetadata; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.UnaryCallable; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.MoreExecutors; import javax.annotation.Nonnull; +import org.threeten.bp.Duration; /** * This callable will: @@ -58,6 +60,11 @@ public ApiFuture futureCall(RequestT request, ApiCallContext context) BigtableTracerUnaryCallback callback = new BigtableTracerUnaryCallback( (BigtableTracer) context.getTracer(), responseMetadata); + GrpcCallContext callContext = (GrpcCallContext) context; + Duration deadline = callContext.getOption(BigtableTracer.OPERATION_TIMEOUT_KEY); + if (deadline != null) { + ((BigtableTracer) context.getTracer()).setOperationTimeout(deadline); + } ApiFuture future = innerCallable.futureCall( request, diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java index d85300828b..62ac0f1153 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java @@ -58,6 +58,7 @@ public class BuiltinMetricsConstants { static final String SERVER_LATENCIES_NAME = "server_latencies"; static final String FIRST_RESPONSE_LATENCIES_NAME = "first_response_latencies"; static final String APPLICATION_BLOCKING_LATENCIES_NAME = "application_latencies"; + static final String REMAINING_DEADLINE_NAME = "remaining_deadline"; static final String CLIENT_BLOCKING_LATENCIES_NAME = "throttling_latencies"; static final String PER_CONNECTION_ERROR_COUNT_NAME = "per_connection_error_count"; @@ -214,6 +215,16 @@ public static Map getAllViews() { ImmutableSet.builder() .add(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY, CLIENT_NAME_KEY) .build()); + defineView( + views, + REMAINING_DEADLINE_NAME, + AGGREGATION_WITH_MILLIS_HISTOGRAM, + InstrumentType.HISTOGRAM, + "ms", + ImmutableSet.builder() + .addAll(COMMON_ATTRIBUTES) + .add(STREAMING_KEY, STATUS_KEY) + .build()); return views.build(); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java index 14a112b270..07583bde52 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java @@ -37,6 +37,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nullable; import org.threeten.bp.Duration; @@ -46,6 +48,8 @@ */ class BuiltinMetricsTracer extends BigtableTracer { + private static final Logger logger = Logger.getLogger(BuiltinMetricsTracer.class.getName()); + private static final String NAME = "java-bigtable/" + Version.VERSION; private final OperationType operationType; private final SpanName spanName; @@ -85,6 +89,9 @@ class BuiltinMetricsTracer extends BigtableTracer { private Long serverLatencies = null; private final AtomicLong grpcMessageSentDelay = new AtomicLong(0); + private Duration operationTimeout = Duration.ofMillis(0); + private long remainingOperationTimeout = 0; + // OpenCensus (and server) histogram buckets use [start, end), however OpenTelemetry uses (start, // end]. To work around this, we measure all the latencies in nanoseconds and convert them // to milliseconds and use DoubleHistogram. This should minimize the chance of a data @@ -95,6 +102,7 @@ class BuiltinMetricsTracer extends BigtableTracer { private final DoubleHistogram firstResponseLatenciesHistogram; private final DoubleHistogram clientBlockingLatenciesHistogram; private final DoubleHistogram applicationBlockingLatenciesHistogram; + private final DoubleHistogram remainingDeadlineHistogram; private final LongCounter connectivityErrorCounter; private final LongCounter retryCounter; @@ -108,6 +116,7 @@ class BuiltinMetricsTracer extends BigtableTracer { DoubleHistogram firstResponseLatenciesHistogram, DoubleHistogram clientBlockingLatenciesHistogram, DoubleHistogram applicationBlockingLatenciesHistogram, + DoubleHistogram deadlineHistogram, LongCounter connectivityErrorCounter, LongCounter retryCounter) { this.operationType = operationType; @@ -120,6 +129,7 @@ class BuiltinMetricsTracer extends BigtableTracer { this.firstResponseLatenciesHistogram = firstResponseLatenciesHistogram; this.clientBlockingLatenciesHistogram = clientBlockingLatenciesHistogram; this.applicationBlockingLatenciesHistogram = applicationBlockingLatenciesHistogram; + this.remainingDeadlineHistogram = deadlineHistogram; this.connectivityErrorCounter = connectivityErrorCounter; this.retryCounter = retryCounter; } @@ -167,6 +177,11 @@ public void attemptStarted(Object request, int attemptNumber) { } } } + // OperationTimeout is only set after the first attempt. + if (attemptCount > 1) { + remainingOperationTimeout = + operationTimeout.toMillis() - operationTimer.elapsed(TimeUnit.MILLISECONDS); + } } @Override @@ -266,6 +281,14 @@ public void grpcMessageSent() { grpcMessageSentDelay.set(attemptTimer.elapsed(TimeUnit.NANOSECONDS)); } + /* + This is called by BigtableTracerCallables that sets operation timeout from user settings. + */ + @Override + public void setOperationTimeout(Duration operationTimeout) { + this.operationTimeout = operationTimeout; + } + @Override public void disableFlowControl() { flowControlIsDisabled = true; @@ -355,6 +378,17 @@ private void recordAttemptCompletion(@Nullable Throwable status) { attemptLatenciesHistogram.record( convertToMs(attemptTimer.elapsed(TimeUnit.NANOSECONDS)), attributes); + if (attemptCount <= 1) { + remainingDeadlineHistogram.record(operationTimeout.toMillis(), attributes); + } else if (remainingOperationTimeout >= 0) { + remainingDeadlineHistogram.record(remainingOperationTimeout, attributes); + } else if (operationTimeout.toMillis() != 0) { + // If the operationTimeout is set but remaining deadline is < 0, log a warning. This should + // never happen. + logger.log( + Level.WARNING, "The remaining deadline was less than 0: " + remainingOperationTimeout); + } + if (serverLatencies != null) { serverLatenciesHistogram.record(serverLatencies, attributes); connectivityErrorCounter.add(0, attributes); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java index f0ac656978..18d3a3ace9 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java @@ -22,6 +22,7 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.FIRST_RESPONSE_LATENCIES_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME; +import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME; @@ -55,6 +56,7 @@ public class BuiltinMetricsTracerFactory extends BaseApiTracerFactory { private final DoubleHistogram firstResponseLatenciesHistogram; private final DoubleHistogram clientBlockingLatenciesHistogram; private final DoubleHistogram applicationBlockingLatenciesHistogram; + private final DoubleHistogram remainingDeadlineHistogram; private final LongCounter connectivityErrorCounter; private final LongCounter retryCounter; @@ -108,6 +110,13 @@ public static BuiltinMetricsTracerFactory create( "The latency of the client application consuming available response data.") .setUnit(MILLISECOND) .build(); + remainingDeadlineHistogram = + meter + .histogramBuilder(REMAINING_DEADLINE_NAME) + .setDescription( + "The remaining deadline when the request is sent to grpc. This will either be the operation timeout, or the remaining deadline from operation timeout after retries and back offs.") + .setUnit(MILLISECOND) + .build(); connectivityErrorCounter = meter .counterBuilder(CONNECTIVITY_ERROR_COUNT_NAME) @@ -135,6 +144,7 @@ public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType op firstResponseLatenciesHistogram, clientBlockingLatenciesHistogram, applicationBlockingLatenciesHistogram, + remainingDeadlineHistogram, connectivityErrorCounter, retryCounter); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java index d89aa90c6b..6135b2347f 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java @@ -232,4 +232,11 @@ public void grpcMessageSent() { tracer.grpcMessageSent(); } } + + @Override + public void setOperationTimeout(Duration operationTimeout) { + for (BigtableTracer tracer : bigtableTracers) { + tracer.setOperationTimeout(operationTimeout); + } + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index cec518e2c3..e902f0271f 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -24,6 +24,7 @@ import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.FIRST_RESPONSE_LATENCIES_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METHOD_KEY; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME; +import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME; import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.STATUS_KEY; @@ -90,11 +91,13 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; import io.opentelemetry.sdk.metrics.data.MetricData; import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import java.io.IOException; import java.net.SocketAddress; import java.nio.charset.Charset; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -103,6 +106,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import javax.annotation.Nullable; import org.junit.After; import org.junit.Assert; @@ -213,6 +217,17 @@ public void sendHeaders(Metadata headers) { .retrySettings() .setInitialRetryDelayDuration(java.time.Duration.ofMillis(200)); + stubSettingsBuilder + .readRowsSettings() + .retrySettings() + .setTotalTimeoutDuration(Duration.ofMillis(9000)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(6000)) + .setRpcTimeoutMultiplier(1) + .setInitialRpcTimeoutDuration(Duration.ofMillis(6000)) + .setInitialRetryDelayDuration(Duration.ofMillis(10)) + .setRetryDelayMultiplier(1) + .setMaxRetryDelayDuration(Duration.ofMillis(10)); + stubSettingsBuilder .bulkMutateRowsSettings() .setBatchingSettings( @@ -750,6 +765,53 @@ public void testPermanentFailure() { verifyAttributes(opLatency, expected); } + @Test + public void testRemainingDeadline() { + stub.readRowsCallable().all().call(Query.create(TABLE)); + MetricData deadlineMetric = getMetricData(metricReader, REMAINING_DEADLINE_NAME); + + Attributes retryAttributes = + baseAttributes + .toBuilder() + .put(STATUS_KEY, "UNAVAILABLE") + .put(TABLE_ID_KEY, TABLE) + .put(METHOD_KEY, "Bigtable.ReadRows") + .put(ZONE_ID_KEY, "global") + .put(CLUSTER_ID_KEY, "unspecified") + .put(STREAMING_KEY, true) + .put(CLIENT_NAME_KEY, CLIENT_NAME) + .build(); + HistogramPointData retryHistogramPointData = + deadlineMetric.getHistogramData().getPoints().stream() + .filter(pd -> pd.getAttributes().equals(retryAttributes)) + .collect(Collectors.toList()) + .get(0); + + double retryRemainingDeadline = retryHistogramPointData.getSum(); + // The retry remaining deadline should be equivalent to the original timeout. + assertThat(retryRemainingDeadline).isEqualTo(9000); + + Attributes okAttributes = + baseAttributes + .toBuilder() + .put(STATUS_KEY, "OK") + .put(TABLE_ID_KEY, TABLE) + .put(ZONE_ID_KEY, ZONE) + .put(CLUSTER_ID_KEY, CLUSTER) + .put(METHOD_KEY, "Bigtable.ReadRows") + .put(STREAMING_KEY, true) + .put(CLIENT_NAME_KEY, CLIENT_NAME) + .build(); + HistogramPointData okHistogramPointData = + deadlineMetric.getHistogramData().getPoints().stream() + .filter(pd -> pd.getAttributes().equals(okAttributes)) + .collect(Collectors.toList()) + .get(0); + + double okRemainingDeadline = okHistogramPointData.getSum(); + assertThat(okRemainingDeadline).isWithin(200).of(8500); + } + private static class FakeService extends BigtableGrpc.BigtableImplBase { static List createFakeResponse() {