feat(Datastore): Introduce Client Side Metrics (#12718)

Author: lqiu96

java-datastore/google-cloud-datastore/pom.xml

@@ -79,6 +79,10 @@
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java-util</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.api</groupId>
       <artifactId>gax</artifactId>
@@ -196,12 +200,20 @@
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk</artifactId>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-common</artifactId>
-      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-monitoring</artifactId>
+      <version>3.91.0</version><!-- {x-version-update:google-cloud-monitoring:current} -->
+    </dependency>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-cloud-monitoring-v3</artifactId>
+      <version>3.91.0</version><!-- {x-version-update:google-cloud-monitoring:current} -->
     </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
@@ -216,7 +228,6 @@
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-metrics</artifactId>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>

java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java

@@ -47,6 +47,7 @@
 import com.google.cloud.ServiceOptions;
 import com.google.cloud.datastore.execution.AggregationQueryExecutor;
 import com.google.cloud.datastore.spi.v1.DatastoreRpc;
+import com.google.cloud.datastore.telemetry.BuiltInDatastoreMetricsProvider;
 import com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder;
 import com.google.cloud.datastore.telemetry.TelemetryConstants;
 import com.google.cloud.datastore.telemetry.TelemetryUtils;
@@ -69,7 +70,9 @@
 import com.google.datastore.v1.TransactionOptions;
 import com.google.protobuf.ByteString;
 import io.grpc.Status;
+import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -98,7 +101,9 @@ final class DatastoreImpl extends BaseService<DatastoreOptions> implements Datas
 
   private final com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil =
       getOptions().getTraceUtil();
-  private final DatastoreMetricsRecorder metricsRecorder = getOptions().getMetricsRecorder();
+  private final DatastoreMetricsRecorder metricsRecorder;
+  private final OpenTelemetry builtInOpenTelemetry;
+
   private final ReadOptionProtoPreparer readOptionProtoPreparer;
   private final AggregationQueryExecutor aggregationQueryExecutor;
 
@@ -107,6 +112,8 @@ final class DatastoreImpl extends BaseService<DatastoreOptions> implements Datas
     this.datastoreRpc = options.getDatastoreRpcV1();
     retrySettings =
         MoreObjects.firstNonNull(options.getRetrySettings(), ServiceOptions.getNoRetrySettings());
+    builtInOpenTelemetry = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options);
+    metricsRecorder = DatastoreMetricsRecorder.getInstance(options, builtInOpenTelemetry);
 
     readOptionProtoPreparer = new ReadOptionProtoPreparer();
     aggregationQueryExecutor =
@@ -162,13 +169,31 @@ public T call() throws DatastoreException {
     }
   }
 
+  /**
+   * Closes the Datastore client and releases all resources.
+   *
+   * <p>This method closes the underlying RPC channel and then closes the {@link
+   * com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder}. For clients using the built-in
+   * Cloud Monitoring exporter, closing the recorder flushes any buffered metrics and shuts down the
+   * private {@link io.opentelemetry.sdk.OpenTelemetrySdk} instance. For clients using a
+   * user-provided {@link io.opentelemetry.api.OpenTelemetry} instance, the recorder close is a
+   * no-op since the user owns that instance's lifecycle.
+   */
   @Override
   public void close() throws Exception {
     try {
       datastoreRpc.close();
     } catch (Exception e) {
       logger.log(Level.WARNING, "Failed to close channels", e);
     }
+    // Shut down the built-in OTel SDK as we manage its lifecycle
+    if (builtInOpenTelemetry instanceof OpenTelemetrySdk) {
+      try {
+        ((OpenTelemetrySdk) builtInOpenTelemetry).close();
+      } catch (Exception e) {
+        logger.log(Level.WARNING, "Failed to close built-in OpenTelemetry SDK instance.", e);
+      }
+    }
   }
 
   @Override
@@ -242,7 +267,9 @@ public T call() throws DatastoreException {
     private void recordAttempt(String status) {
       Map<String, String> attributes =
           TelemetryUtils.buildMetricAttributes(
-              TelemetryConstants.METHOD_TRANSACTION_COMMIT, status);
+              TelemetryConstants.METHOD_TRANSACTION_COMMIT,
+              status,
+              datastore.getOptions().getDatabaseId());
       metricsRecorder.recordTransactionAttemptCount(1, attributes);
     }
   }
@@ -280,7 +307,8 @@ public <T> T runInTransaction(
     } finally {
       long latencyMs = stopwatch.elapsed(TimeUnit.MILLISECONDS);
       Map<String, String> attributes =
-          TelemetryUtils.buildMetricAttributes(TelemetryConstants.METHOD_TRANSACTION_RUN, status);
+          TelemetryUtils.buildMetricAttributes(
+              TelemetryConstants.METHOD_TRANSACTION_RUN, status, getOptions().getDatabaseId());
       metricsRecorder.recordTransactionLatency(latencyMs, attributes);
       span.end();
     }
@@ -789,7 +817,8 @@ private <T> T runWithObservability(
 
     DatastoreOptions options = getOptions();
     Callable<T> attemptCallable =
-        TelemetryUtils.attemptMetricsCallable(callable, metricsRecorder, methodName);
+        TelemetryUtils.attemptMetricsCallable(
+            callable, metricsRecorder, methodName, options.getDatabaseId());
     try (TraceUtil.Scope ignored = span.makeCurrent()) {
       return RetryHelper.runWithRetries(
           attemptCallable, retrySettings, exceptionHandler, options.getClock());
@@ -799,7 +828,11 @@ private <T> T runWithObservability(
       throw DatastoreException.translateAndThrow(e);
     } finally {
       TelemetryUtils.recordOperationMetrics(
-          metricsRecorder, operationStopwatch, methodName, operationStatus);
+          metricsRecorder,
+          operationStopwatch,
+          methodName,
+          operationStatus,
+          options.getDatabaseId());
       span.end();
     }
   }

java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java

@@ -16,25 +16,39 @@
 
 package com.google.cloud.datastore;
 
+import com.google.api.core.BetaApi;
 import io.opentelemetry.api.OpenTelemetry;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+/**
+ * Represents the options that are used to configure the use of OpenTelemetry for telemetry
+ * collection in the Datastore SDK.
+ */
 public class DatastoreOpenTelemetryOptions {
   private final boolean tracingEnabled;
   private final boolean metricsEnabled;
+  private final boolean exportBuiltinMetricsToGoogleCloudMonitoring;
   private final @Nullable OpenTelemetry openTelemetry;
 
   DatastoreOpenTelemetryOptions(Builder builder) {
     this.tracingEnabled = builder.tracingEnabled;
     this.metricsEnabled = builder.metricsEnabled;
+    this.exportBuiltinMetricsToGoogleCloudMonitoring =
+        builder.exportBuiltinMetricsToGoogleCloudMonitoring;
     this.openTelemetry = builder.openTelemetry;
   }
 
   /**
-   * Returns whether either tracing or metrics are enabled. Telemetry is disabled by default.
+   * Returns whether either tracing or custom metrics (via a user-provided {@link OpenTelemetry}
+   * instance) are enabled.
+   *
+   * <p><b>Note:</b> This method does <em>not</em> reflect the state of built-in metrics export to
+   * Google Cloud Monitoring, which is controlled separately by {@link
+   * #isExportBuiltinMetricsToGoogleCloudMonitoring()} and is {@code false} by default. To check
+   * whether any telemetry is active, also consult that flag.
    *
-   * @return {@code true} if either tracing or metrics are enabled, {@code false} otherwise.
+   * @return {@code true} if tracing or custom OTel metrics are enabled, {@code false} otherwise.
    */
   public boolean isEnabled() {
     return tracingEnabled || metricsEnabled;
@@ -50,24 +64,54 @@ public boolean isTracingEnabled() {
   }
 
   /**
-   * Returns whether metrics are enabled.
+   * Returns whether metrics are enabled for the custom (user-provided) OpenTelemetry backend.
    *
    * @return {@code true} if metrics are enabled, {@code false} otherwise.
    */
   public boolean isMetricsEnabled() {
     return metricsEnabled;
   }
 
+  /**
+   * Returns whether built-in metrics should be exported to Google Cloud Monitoring.
+   *
+   * <p>When enabled, client-side metrics are automatically exported to Google Cloud Monitoring
+   * using the Cloud Monitoring API. This is independent of the custom OpenTelemetry backend
+   * configured via {@link #getOpenTelemetry()}.
+   *
+   * @return {@code true} if built-in metrics export to Cloud Monitoring is enabled, {@code false}
+   *     otherwise.
+   */
+  @BetaApi
+  public boolean isExportBuiltinMetricsToGoogleCloudMonitoring() {
+    return exportBuiltinMetricsToGoogleCloudMonitoring;
+  }
+
+  /**
+   * Returns the custom {@link OpenTelemetry} instance, if one was provided.
+   *
+   * @return the custom {@link OpenTelemetry} instance, or {@code null} if none was provided.
+   */
   @Nullable
   public OpenTelemetry getOpenTelemetry() {
     return openTelemetry;
   }
 
+  /**
+   * Returns a new {@link Builder} initialized with the values from this options instance.
+   *
+   * @return a new {@link Builder}.
+   */
   @Nonnull
   public DatastoreOpenTelemetryOptions.Builder toBuilder() {
     return new DatastoreOpenTelemetryOptions.Builder(this);
   }
 
+  /**
+   * Returns a new default {@link Builder}.
+   *
+   * @return a new {@link Builder}.
+   */
   @Nonnull
   public static DatastoreOpenTelemetryOptions.Builder newBuilder() {
     return new DatastoreOpenTelemetryOptions.Builder();
@@ -77,25 +121,31 @@ public static class Builder {
 
     private boolean tracingEnabled;
     private boolean metricsEnabled;
+    private boolean exportBuiltinMetricsToGoogleCloudMonitoring;
 
     @Nullable private OpenTelemetry openTelemetry;
 
     private Builder() {
       tracingEnabled = false;
       metricsEnabled = false;
+      // TODO(b/405457573): This is disabled by default until the Firestore namespace is deployed
+      exportBuiltinMetricsToGoogleCloudMonitoring = false;
       openTelemetry = null;
     }
 
     private Builder(DatastoreOpenTelemetryOptions options) {
       this.tracingEnabled = options.tracingEnabled;
       this.metricsEnabled = options.metricsEnabled;
+      this.exportBuiltinMetricsToGoogleCloudMonitoring =
+          options.exportBuiltinMetricsToGoogleCloudMonitoring;
       this.openTelemetry = options.openTelemetry;
     }
 
     /**
      * Sets whether tracing should be enabled.
      *
      * @param enabled Whether tracing should be enabled.
+     * @return this builder instance.
      */
     @Nonnull
     public DatastoreOpenTelemetryOptions.Builder setTracingEnabled(boolean enabled) {
@@ -104,23 +154,41 @@ public DatastoreOpenTelemetryOptions.Builder setTracingEnabled(boolean enabled)
     }
 
     /**
-     * Sets whether metrics should be enabled.
+     * Sets whether metrics should be enabled for the custom (user-provided) OpenTelemetry backend.
      *
      * @param enabled Whether metrics should be enabled.
-     * @return the builder instance.
+     * @return this builder instance.
      */
     @Nonnull
     DatastoreOpenTelemetryOptions.Builder setMetricsEnabled(boolean enabled) {
       this.metricsEnabled = enabled;
       return this;
     }
 
+    /**
+     * Sets whether built-in metrics should be exported to Google Cloud Monitoring.
+     *
+     * <p>When enabled, client-side metrics are automatically exported to Google Cloud Monitoring
+     * using the Cloud Monitoring API. This can be disabled to prevent metrics from being sent to
+     * Cloud Monitoring while still allowing metrics to flow to a custom OpenTelemetry backend.
+     *
+     * @param exportBuiltinMetrics Whether built-in metrics should be exported to Cloud Monitoring.
+     * @return this builder instance.
+     */
+    @BetaApi
+    public DatastoreOpenTelemetryOptions.Builder setExportBuiltinMetricsToGoogleCloudMonitoring(
+        boolean exportBuiltinMetrics) {
+      this.exportBuiltinMetricsToGoogleCloudMonitoring = exportBuiltinMetrics;
+      return this;
+    }
+
     /**
      * Sets the {@link OpenTelemetry} to use with this Datastore instance. If telemetry collection
-     * is enabled, but an `OpenTelemetry` is not provided, the Datastore SDK will attempt to use the
-     * `GlobalOpenTelemetry`.
+     * is enabled, but an {@code OpenTelemetry} is not provided, the Datastore SDK will attempt to
+     * use the {@code GlobalOpenTelemetry}.
      *
      * @param openTelemetry The OpenTelemetry that should be used by this Datastore instance.
+     * @return this builder instance.
      */
     @Nonnull
     public DatastoreOpenTelemetryOptions.Builder setOpenTelemetry(
@@ -129,6 +197,11 @@ public DatastoreOpenTelemetryOptions.Builder setOpenTelemetry(
       return this;
     }
 
+    /**
+     * Builds a new {@link DatastoreOpenTelemetryOptions} instance from this builder.
+     *
+     * @return a new {@link DatastoreOpenTelemetryOptions}.
+     */
     @Nonnull
     public DatastoreOpenTelemetryOptions build() {
       return new DatastoreOpenTelemetryOptions(this);

java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java

@@ -31,7 +31,6 @@
 import com.google.cloud.datastore.spi.v1.DatastoreRpc;
 import com.google.cloud.datastore.spi.v1.GrpcDatastoreRpc;
 import com.google.cloud.datastore.spi.v1.HttpDatastoreRpc;
-import com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder;
 import com.google.cloud.datastore.v1.DatastoreSettings;
 import com.google.cloud.grpc.GrpcTransportOptions;
 import com.google.cloud.http.HttpTransportOptions;
@@ -82,7 +81,6 @@ public class DatastoreOptions extends ServiceOptions<Datastore, DatastoreOptions
 
   private final transient @Nonnull DatastoreOpenTelemetryOptions openTelemetryOptions;
   private final transient @Nonnull com.google.cloud.datastore.telemetry.TraceUtil traceUtil;
-  private final transient @Nonnull DatastoreMetricsRecorder metricsRecorder;
 
   public static class DefaultDatastoreFactory implements DatastoreFactory {
 
@@ -123,11 +121,6 @@ public DatastoreOpenTelemetryOptions getOpenTelemetryOptions() {
     return openTelemetryOptions;
   }
 
-  @Nonnull
-  DatastoreMetricsRecorder getMetricsRecorder() {
-    return metricsRecorder;
-  }
-
   public static class Builder extends ServiceOptions.Builder<Datastore, DatastoreOptions, Builder> {
 
     private String namespace;
@@ -240,7 +233,6 @@ private DatastoreOptions(Builder builder) {
             ? builder.openTelemetryOptions
             : DatastoreOpenTelemetryOptions.newBuilder().build();
     this.traceUtil = com.google.cloud.datastore.telemetry.TraceUtil.getInstance(this);
-    this.metricsRecorder = DatastoreMetricsRecorder.getInstance(this);
 
     namespace = MoreObjects.firstNonNull(builder.namespace, defaultNamespace());
     databaseId = MoreObjects.firstNonNull(builder.databaseId, DEFAULT_DATABASE_ID);