feat: add internal "deadline remaining" client side metric #2341

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java

@@ -23,6 +23,7 @@
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.PER_CONNECTION_ERROR_COUNT_NAME;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME;
 
@@ -115,7 +116,8 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter {
               CLIENT_BLOCKING_LATENCIES_NAME,
               APPLICATION_BLOCKING_LATENCIES_NAME,
               RETRY_COUNT_NAME,
-              CONNECTIVITY_ERROR_COUNT_NAME)
+              CONNECTIVITY_ERROR_COUNT_NAME,
+              REMAINING_DEADLINE_NAME)
           .stream()
           .map(m -> METER_NAME + m)
           .collect(ImmutableList.toImmutableList());

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableGrpcStreamTracer.java

@@ -17,6 +17,7 @@
 
 import io.grpc.ClientStreamTracer;
 import io.grpc.Metadata;
+import org.threeten.bp.Duration;
 
 /**
  * Records the time a request is enqueued in a grpc channel queue. This a bridge between gRPC stream
@@ -26,28 +27,33 @@
 class BigtableGrpcStreamTracer extends ClientStreamTracer {
 
   private final BigtableTracer tracer;
+  private final Duration deadline;
 
-  public BigtableGrpcStreamTracer(BigtableTracer tracer) {
+  public BigtableGrpcStreamTracer(BigtableTracer tracer, Duration deadline) {
     this.tracer = tracer;
+    this.deadline = deadline;
   }
 
   @Override
   public void outboundMessageSent(int seqNo, long optionalWireSize, long optionalUncompressedSize) {
     tracer.grpcMessageSent();
+    tracer.setRemainingDeadline(deadline.toMillis());
   }
 
   static class Factory extends ClientStreamTracer.Factory {
 
     private final BigtableTracer tracer;
+    private final Duration deadline;
 
-    Factory(BigtableTracer tracer) {
+    Factory(BigtableTracer tracer, Duration deadline) {
       this.tracer = tracer;
+      this.deadline = deadline;
     }
 
     @Override
     public ClientStreamTracer newClientStreamTracer(
         ClientStreamTracer.StreamInfo info, Metadata headers) {
-      return new BigtableGrpcStreamTracer(tracer);
+      return new BigtableGrpcStreamTracer(tracer, deadline);
     }
   }
 }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracer.java

@@ -93,4 +93,12 @@ public void grpcChannelQueuedLatencies(long queuedTimeMs) {
   public void grpcMessageSent() {
     // noop
   }
+
+  /**
+   * Set the remaining customer specified deadline so it can be exported in a metric. This will be
+   * called in BuiltinMetricsTracer.
+   */
+  public void setRemainingDeadline(long deadline) {
+    // noop
+  }
 }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsConstants.java

@@ -58,6 +58,7 @@ public class BuiltinMetricsConstants {
   static final String SERVER_LATENCIES_NAME = "server_latencies";
   static final String FIRST_RESPONSE_LATENCIES_NAME = "first_response_latencies";
   static final String APPLICATION_BLOCKING_LATENCIES_NAME = "application_latencies";
+  static final String REMAINING_DEADLINE_NAME = "remaining_deadline";
   static final String CLIENT_BLOCKING_LATENCIES_NAME = "throttling_latencies";
   static final String PER_CONNECTION_ERROR_COUNT_NAME = "per_connection_error_count";
 
@@ -214,6 +215,16 @@ public static Map<InstrumentSelector, View> getAllViews() {
         ImmutableSet.<AttributeKey>builder()
             .add(BIGTABLE_PROJECT_ID_KEY, INSTANCE_ID_KEY, APP_PROFILE_KEY, CLIENT_NAME_KEY)
             .build());
+    defineView(
+        views,
+        REMAINING_DEADLINE_NAME,
+        AGGREGATION_WITH_MILLIS_HISTOGRAM,
+        InstrumentType.HISTOGRAM,
+        "ms",
+        ImmutableSet.<AttributeKey>builder()
+            .addAll(COMMON_ATTRIBUTES)
+            .add(STREAMING_KEY, STATUS_KEY)
+            .build());
 
     return views.build();
   }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracer.java

@@ -86,6 +86,8 @@ class BuiltinMetricsTracer extends BigtableTracer {
   private Long serverLatencies = null;
   private final AtomicLong grpcMessageSentDelay = new AtomicLong(0);
 
+  private long deadlineRemaining = 0;
+
   // OpenCensus (and server) histogram buckets use [start, end), however OpenTelemetry uses (start,
   // end]. To work around this, we measure all the latencies in nanoseconds and convert them
   // to milliseconds and use DoubleHistogram. This should minimize the chance of a data
@@ -96,6 +98,7 @@ class BuiltinMetricsTracer extends BigtableTracer {
   private final DoubleHistogram firstResponseLatenciesHistogram;
   private final DoubleHistogram clientBlockingLatenciesHistogram;
   private final DoubleHistogram applicationBlockingLatenciesHistogram;
+  private final DoubleHistogram remainingDeadlineHistogram;
   private final LongCounter connectivityErrorCounter;
   private final LongCounter retryCounter;
 
@@ -109,6 +112,7 @@ class BuiltinMetricsTracer extends BigtableTracer {
       DoubleHistogram firstResponseLatenciesHistogram,
       DoubleHistogram clientBlockingLatenciesHistogram,
       DoubleHistogram applicationBlockingLatenciesHistogram,
+      DoubleHistogram remainingDeadlineHistogram,
       LongCounter connectivityErrorCounter,
       LongCounter retryCounter) {
     this.operationType = operationType;
@@ -121,6 +125,7 @@ class BuiltinMetricsTracer extends BigtableTracer {
     this.firstResponseLatenciesHistogram = firstResponseLatenciesHistogram;
     this.clientBlockingLatenciesHistogram = clientBlockingLatenciesHistogram;
     this.applicationBlockingLatenciesHistogram = applicationBlockingLatenciesHistogram;
+    this.remainingDeadlineHistogram = remainingDeadlineHistogram;
     this.connectivityErrorCounter = connectivityErrorCounter;
     this.retryCounter = retryCounter;
   }
@@ -268,6 +273,12 @@ public void grpcMessageSent() {
     grpcMessageSentDelay.set(attemptTimer.elapsed(TimeUnit.NANOSECONDS));
   }
 
+  @Override
+  public void setRemainingDeadline(long deadline) {
+    long timeElapsed = operationTimer.elapsed(TimeUnit.MILLISECONDS);
+    deadlineRemaining = deadline - timeElapsed;
+  }
+
   @Override
   public void disableFlowControl() {
     flowControlIsDisabled = true;
@@ -306,6 +317,8 @@ private void recordOperationCompletion(@Nullable Throwable status) {
 
     operationLatenciesHistogram.record(convertToMs(operationLatencyNano), attributes);
 
+    remainingDeadlineHistogram.record(deadlineRemaining, attributes);
+
     // serverLatencyTimer should already be stopped in recordAttemptCompletion
     long applicationLatencyNano = operationLatencyNano - totalServerLatencyNano.get();
     applicationBlockingLatenciesHistogram.record(convertToMs(applicationLatencyNano), attributes);

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerFactory.java

@@ -22,6 +22,7 @@
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.FIRST_RESPONSE_LATENCIES_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METER_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME;
 
@@ -55,6 +56,7 @@ public class BuiltinMetricsTracerFactory extends BaseApiTracerFactory {
   private final DoubleHistogram firstResponseLatenciesHistogram;
   private final DoubleHistogram clientBlockingLatenciesHistogram;
   private final DoubleHistogram applicationBlockingLatenciesHistogram;
+  private final DoubleHistogram remainingDeadlineHistogram;
   private final LongCounter connectivityErrorCounter;
   private final LongCounter retryCounter;
 
@@ -108,6 +110,12 @@ public static BuiltinMetricsTracerFactory create(
                 "The latency of the client application consuming available response data.")
             .setUnit(MILLISECOND)
             .build();
+    remainingDeadlineHistogram =
+        meter
+            .histogramBuilder(REMAINING_DEADLINE_NAME)
+            .setDescription("The remaining customer specified deadline at the end of the request.")
+            .setUnit(MILLISECOND)
+            .build();
     connectivityErrorCounter =
         meter
             .counterBuilder(CONNECTIVITY_ERROR_COUNT_NAME)
@@ -135,6 +143,7 @@ public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType op
         firstResponseLatenciesHistogram,
         clientBlockingLatenciesHistogram,
         applicationBlockingLatenciesHistogram,
+        remainingDeadlineHistogram,
         connectivityErrorCounter,
         retryCounter);
   }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/CompositeTracer.java

@@ -232,4 +232,11 @@ public void grpcMessageSent() {
       tracer.grpcMessageSent();
     }
   }
+
+  @Override
+  public void setRemainingDeadline(long deadline) {
+    for (BigtableTracer tracer : bigtableTracers) {
+      tracer.setRemainingDeadline(deadline);
+    }
+  }
 }

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java

@@ -50,6 +50,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
+import org.threeten.bp.Duration;
 
 /** Utilities to help integrating with OpenCensus. */
 @InternalApi("For internal use only")
@@ -222,9 +223,11 @@ static GrpcCallContext injectBigtableStreamTracer(
     if (context instanceof GrpcCallContext) {
       GrpcCallContext callContext = (GrpcCallContext) context;
       CallOptions callOptions = callContext.getCallOptions();
+      Duration deadline = callContext.getTimeout();
       return responseMetadata.addHandlers(
           callContext.withCallOptions(
-              callOptions.withStreamTracerFactory(new BigtableGrpcStreamTracer.Factory(tracer))));
+              callOptions.withStreamTracerFactory(
+                  new BigtableGrpcStreamTracer.Factory(tracer, deadline))));
     } else {
       // context should always be an instance of GrpcCallContext. If not throw an exception
       // so we can see what class context is.

google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java

@@ -23,6 +23,7 @@
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.CONNECTIVITY_ERROR_COUNT_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.METHOD_KEY;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.OPERATION_LATENCIES_NAME;
+import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.REMAINING_DEADLINE_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.RETRY_COUNT_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.SERVER_LATENCIES_NAME;
 import static com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsConstants.STATUS_KEY;
@@ -94,6 +95,7 @@
 import java.io.IOException;
 import java.net.SocketAddress;
 import java.nio.charset.Charset;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
@@ -211,6 +213,11 @@ public void sendHeaders(Metadata headers) {
         .retrySettings()
         .setInitialRetryDelayDuration(java.time.Duration.ofMillis(200));
 
+    stubSettingsBuilder
+        .readRowsSettings()
+        .retrySettings()
+        .setTotalTimeoutDuration(Duration.ofMillis(3000));
+
     stubSettingsBuilder
         .bulkMutateRowsSettings()
         .setBatchingSettings(
@@ -702,6 +709,27 @@ public void testPermanentFailure() {
     verifyAttributes(opLatency, expected);
   }
 
+  @Test
+  public void testRemainingDeadline() {
+    stub.readRowsCallable().all().call(Query.create(TABLE));
+    MetricData remainingDeadlineMetric = getMetricData(metricReader, REMAINING_DEADLINE_NAME);
+
+    Attributes attributes =
+        baseAttributes
+            .toBuilder()
+            .put(STATUS_KEY, "OK")
+            .put(TABLE_ID_KEY, TABLE)
+            .put(ZONE_ID_KEY, ZONE)
+            .put(CLUSTER_ID_KEY, CLUSTER)
+            .put(METHOD_KEY, "Bigtable.ReadRows")
+            .put(STREAMING_KEY, true)
+            .put(CLIENT_NAME_KEY, CLIENT_NAME)
+            .build();
+
+    long remainingDeadline = getAggregatedValue(remainingDeadlineMetric, attributes);
+    assertThat(remainingDeadline).isIn(Range.closed((long) 2000, (long) 2500));
+  }
+
   private static class FakeService extends BigtableGrpc.BigtableImplBase {
 
     static List<ReadRowsResponse> createFakeResponse() {