From d84aae47166fa71d8ca22f6bb54d0b113e84839c Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 8 May 2024 11:14:05 -0700 Subject: [PATCH 01/41] feat: Adding TraceUtil interface and its implementation to enable Tracing controls via DatastoreOptions (#1431) * Adding EnabledTraceUtil, DisabledTraceUtil and TraceUtilTest * Annotating DatastoreOpenTelemetryOptions to be transient as they're not serializable * Adding google-auth-library-credentials dependency due to https://github.com/googleapis/java-datastore/actions/runs/8944472794/job/24571458116?pr=1431 --- google-cloud-datastore/pom.xml | 24 ++ .../DatastoreOpenTelemetryOptions.java | 97 ++++++ .../cloud/datastore/DatastoreOptions.java | 40 +++ .../telemetry/DisabledTraceUtil.java | 109 +++++++ .../datastore/telemetry/EnabledTraceUtil.java | 306 ++++++++++++++++++ .../cloud/datastore/telemetry/TraceUtil.java | 129 ++++++++ .../cloud/datastore/DatastoreOptionsTest.java | 14 + .../telemetry/DisabledTraceUtilTest.java | 53 +++ .../telemetry/EnabledTraceUtilTest.java | 104 ++++++ .../datastore/telemetry/TraceUtilTest.java | 62 ++++ 10 files changed, 938 insertions(+) create mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java create mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java create mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java create mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java create mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java create mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java create mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/TraceUtilTest.java diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index b06df4c0f..f11be9f56 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -16,6 +16,7 @@ google-cloud-datastore + 1.37.0 @@ -38,6 +39,10 @@ com.google.cloud.datastore datastore-v1-proto-client + + com.google.auth + google-auth-library-credentials + io.grpc grpc-api @@ -111,6 +116,19 @@ jsr305 + + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + + + io.opentelemetry + opentelemetry-context + ${opentelemetry.version} + + + ${project.groupId} @@ -160,6 +178,12 @@ 1.4.3 test + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java new file mode 100644 index 000000000..ac266562e --- /dev/null +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java @@ -0,0 +1,97 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore; + +import io.opentelemetry.api.OpenTelemetry; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class DatastoreOpenTelemetryOptions { + private final boolean enabled; + private final @Nullable OpenTelemetry openTelemetry; + + DatastoreOpenTelemetryOptions(Builder builder) { + this.enabled = builder.enabled; + this.openTelemetry = builder.openTelemetry; + } + + public boolean isEnabled() { + return enabled; + } + + @Nullable + public OpenTelemetry getOpenTelemetry() { + return openTelemetry; + } + + @Nonnull + public DatastoreOpenTelemetryOptions.Builder toBuilder() { + return new DatastoreOpenTelemetryOptions.Builder(this); + } + + @Nonnull + public static DatastoreOpenTelemetryOptions.Builder newBuilder() { + return new DatastoreOpenTelemetryOptions.Builder(); + } + + public static class Builder { + + private boolean enabled; + + @Nullable private OpenTelemetry openTelemetry; + + private Builder() { + enabled = false; + openTelemetry = null; + } + + private Builder(DatastoreOpenTelemetryOptions options) { + this.enabled = options.enabled; + this.openTelemetry = options.openTelemetry; + } + + @Nonnull + public DatastoreOpenTelemetryOptions build() { + return new DatastoreOpenTelemetryOptions(this); + } + + /** + * Sets whether tracing should be enabled. + * + * @param enabled Whether tracing should be enabled. + */ + @Nonnull + public DatastoreOpenTelemetryOptions.Builder setTracingEnabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + /** + * Sets the {@link OpenTelemetry} to use with this Datastore instance. If telemetry collection + * is enabled, but an `OpenTelemetry` is not provided, the Datastore SDK will attempt to use the + * `GlobalOpenTelemetry`. + * + * @param openTelemetry The OpenTelemetry that should be used by this Datastore instance. + */ + @Nonnull + public DatastoreOpenTelemetryOptions.Builder setOpenTelemetry( + @Nonnull OpenTelemetry openTelemetry) { + this.openTelemetry = openTelemetry; + return this; + } + } +} diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java index 8437c3e22..cef40eedd 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java @@ -18,6 +18,7 @@ import static com.google.cloud.datastore.Validator.validateNamespace; +import com.google.api.core.BetaApi; import com.google.cloud.ServiceDefaults; import com.google.cloud.ServiceOptions; import com.google.cloud.ServiceRpc; @@ -31,6 +32,8 @@ import java.lang.reflect.Method; import java.util.Objects; import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class DatastoreOptions extends ServiceOptions { @@ -43,6 +46,9 @@ public class DatastoreOptions extends ServiceOptions { private String namespace; private String databaseId; + @Nullable private DatastoreOpenTelemetryOptions openTelemetryOptions = null; + private Builder() {} private Builder(DatastoreOptions options) { super(options); namespace = options.namespace; databaseId = options.databaseId; + this.openTelemetryOptions = options.openTelemetryOptions; } @Override @@ -100,10 +120,30 @@ public Builder setDatabaseId(String databaseId) { this.databaseId = databaseId; return this; } + + /** + * Sets the {@link DatastoreOpenTelemetryOptions} to be used for this Firestore instance. + * + * @param openTelemetryOptions The `DatastoreOpenTelemetryOptions` to use. + */ + @BetaApi + @Nonnull + public Builder setOpenTelemetryOptions( + @Nonnull DatastoreOpenTelemetryOptions openTelemetryOptions) { + this.openTelemetryOptions = openTelemetryOptions; + return this; + } } private DatastoreOptions(Builder builder) { super(DatastoreFactory.class, DatastoreRpcFactory.class, builder, new DatastoreDefaults()); + + this.openTelemetryOptions = + builder.openTelemetryOptions != null + ? builder.openTelemetryOptions + : DatastoreOpenTelemetryOptions.newBuilder().build(); + this.traceUtil = com.google.cloud.datastore.telemetry.TraceUtil.getInstance(this); + namespace = MoreObjects.firstNonNull(builder.namespace, defaultNamespace()); databaseId = MoreObjects.firstNonNull(builder.databaseId, DEFAULT_DATABASE_ID); } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java new file mode 100644 index 000000000..21321897d --- /dev/null +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java @@ -0,0 +1,109 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.core.ApiFunction; +import com.google.api.core.ApiFuture; +import com.google.api.core.InternalApi; +import io.grpc.ManagedChannelBuilder; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Tracing utility implementation, used to stub out tracing instrumentation when tracing is + * disabled. + */ +@InternalApi +public class DisabledTraceUtil implements TraceUtil { + + static class Span implements TraceUtil.Span { + @Override + public void end() {} + + @Override + public void end(Throwable error) {} + + @Override + public void endAtFuture(ApiFuture futureValue) {} + + @Override + public TraceUtil.Span addEvent(String name) { + return this; + } + + @Override + public TraceUtil.Span addEvent(String name, Map attributes) { + return this; + } + + @Override + public TraceUtil.Span setAttribute(String key, int value) { + return this; + } + + @Override + public TraceUtil.Span setAttribute(String key, String value) { + return this; + } + + @Override + public Scope makeCurrent() { + return new Scope(); + } + } + + static class Context implements TraceUtil.Context { + @Override + public Scope makeCurrent() { + return new Scope(); + } + } + + static class Scope implements TraceUtil.Scope { + @Override + public void close() {} + } + + @Nullable + @Override + public ApiFunction getChannelConfigurator() { + return null; + } + + @Override + public Span startSpan(String spanName) { + return new Span(); + } + + @Override + public TraceUtil.Span startSpan(String spanName, TraceUtil.Context parent) { + return new Span(); + } + + @Nonnull + @Override + public TraceUtil.Span currentSpan() { + return new Span(); + } + + @Nonnull + @Override + public TraceUtil.Context currentContext() { + return new Context(); + } +} diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java new file mode 100644 index 000000000..8a5b0b36e --- /dev/null +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java @@ -0,0 +1,306 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.core.ApiFunction; +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.api.core.InternalApi; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.common.base.Throwables; +import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Tracing utility implementation, used to stub out tracing instrumentation when tracing is enabled. + */ +@InternalApi +public class EnabledTraceUtil implements TraceUtil { + private final Tracer tracer; + private final OpenTelemetry openTelemetry; + private final DatastoreOptions datastoreOptions; + + EnabledTraceUtil(DatastoreOptions datastoreOptions) { + OpenTelemetry openTelemetry = datastoreOptions.getOpenTelemetryOptions().getOpenTelemetry(); + + // If tracing is enabled, but an OpenTelemetry instance is not provided, fall back + // to using GlobalOpenTelemetry. + if (openTelemetry == null) { + openTelemetry = GlobalOpenTelemetry.get(); + } + + this.datastoreOptions = datastoreOptions; + this.openTelemetry = openTelemetry; + this.tracer = openTelemetry.getTracer(LIBRARY_NAME); + } + + public OpenTelemetry getOpenTelemetry() { + return openTelemetry; + } + + @Override + @Nullable + public ApiFunction getChannelConfigurator() { + // TODO(jimit) Update this to return a gRPC Channel Configurator after gRPC upgrade. + return null; + } + + static class Span implements TraceUtil.Span { + private final io.opentelemetry.api.trace.Span span; + private final String spanName; + + public Span(io.opentelemetry.api.trace.Span span, String spanName) { + this.span = span; + this.spanName = spanName; + } + + /** Ends this span. */ + @Override + public void end() { + span.end(); + } + + /** Ends this span in an error. */ + @Override + public void end(Throwable error) { + span.setStatus(StatusCode.ERROR, error.getMessage()); + span.recordException( + error, + Attributes.builder() + .put("exception.message", error.getMessage()) + .put("exception.type", error.getClass().getName()) + .put("exception.stacktrace", Throwables.getStackTraceAsString(error)) + .build()); + span.end(); + } + + /** + * If an operation ends in the future, its relevant span should end _after_ the future has been + * completed. This method "appends" the span completion code at the completion of the given + * future. In order for telemetry info to be recorded, the future returned by this method should + * be completed. + */ + @Override + public void endAtFuture(ApiFuture futureValue) { + io.opentelemetry.context.Context asyncContext = io.opentelemetry.context.Context.current(); + ApiFutures.addCallback( + futureValue, + new ApiFutureCallback() { + @Override + public void onFailure(Throwable t) { + try (io.opentelemetry.context.Scope scope = asyncContext.makeCurrent()) { + span.addEvent(spanName + " failed."); + end(t); + } + } + + @Override + public void onSuccess(T result) { + try (io.opentelemetry.context.Scope scope = asyncContext.makeCurrent()) { + span.addEvent(spanName + " succeeded."); + end(); + } + } + }); + } + + /** Adds the given event to this span. */ + @Override + public TraceUtil.Span addEvent(String name) { + span.addEvent(name); + return this; + } + + @Override + public TraceUtil.Span addEvent(String name, Map attributes) { + AttributesBuilder attributesBuilder = Attributes.builder(); + attributes.forEach( + (key, value) -> { + if (value instanceof Integer) { + attributesBuilder.put(key, (int) value); + } else if (value instanceof Long) { + attributesBuilder.put(key, (long) value); + } else if (value instanceof Double) { + attributesBuilder.put(key, (double) value); + } else if (value instanceof Float) { + attributesBuilder.put(key, (float) value); + } else if (value instanceof Boolean) { + attributesBuilder.put(key, (boolean) value); + } else if (value instanceof String) { + attributesBuilder.put(key, (String) value); + } else { + // OpenTelemetry APIs do not support any other type. + throw new IllegalArgumentException( + "Unknown attribute type:" + value.getClass().getSimpleName()); + } + }); + span.addEvent(name, attributesBuilder.build()); + return this; + } + + @Override + public TraceUtil.Span setAttribute(String key, int value) { + span.setAttribute(ATTRIBUTE_SERVICE_PREFIX + key, value); + return this; + } + + @Override + public TraceUtil.Span setAttribute(String key, String value) { + span.setAttribute(ATTRIBUTE_SERVICE_PREFIX + key, value); + return this; + } + + @Override + public Scope makeCurrent() { + try (io.opentelemetry.context.Scope scope = span.makeCurrent()) { + return new Scope(scope); + } + } + } + + static class Scope implements TraceUtil.Scope { + private final io.opentelemetry.context.Scope scope; + + Scope(io.opentelemetry.context.Scope scope) { + this.scope = scope; + } + + @Override + public void close() { + scope.close(); + } + } + + static class Context implements TraceUtil.Context { + private final io.opentelemetry.context.Context context; + + Context(io.opentelemetry.context.Context context) { + this.context = context; + } + + @Override + public Scope makeCurrent() { + try (io.opentelemetry.context.Scope scope = context.makeCurrent()) { + return new Scope(scope); + } + } + } + + /** Applies the current Datastore instance settings as attributes to the current Span */ + private SpanBuilder addSettingsAttributesToCurrentSpan(SpanBuilder spanBuilder) { + spanBuilder = + spanBuilder.setAllAttributes( + Attributes.builder() + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.databaseId", + datastoreOptions.getDatabaseId()) + .put(ATTRIBUTE_SERVICE_PREFIX + "settings.host", datastoreOptions.getHost()) + .build()); + + if (datastoreOptions.getCredentials() != null) { + spanBuilder = + spanBuilder.setAttribute( + ATTRIBUTE_SERVICE_PREFIX + "settings.credentials.authenticationType", + datastoreOptions.getCredentials().getAuthenticationType()); + } + + if (datastoreOptions.getRetrySettings() != null) { + spanBuilder = + spanBuilder.setAllAttributes( + Attributes.builder() + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.initialRetryDelay", + datastoreOptions.getRetrySettings().getInitialRetryDelay().toString()) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.maxRetryDelay", + datastoreOptions.getRetrySettings().getMaxRetryDelay().toString()) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.retryDelayMultiplier", + String.valueOf(datastoreOptions.getRetrySettings().getRetryDelayMultiplier())) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.maxAttempts", + String.valueOf(datastoreOptions.getRetrySettings().getMaxAttempts())) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.initialRpcTimeout", + datastoreOptions.getRetrySettings().getInitialRpcTimeout().toString()) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.maxRpcTimeout", + datastoreOptions.getRetrySettings().getMaxRpcTimeout().toString()) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.rpcTimeoutMultiplier", + String.valueOf(datastoreOptions.getRetrySettings().getRpcTimeoutMultiplier())) + .put( + ATTRIBUTE_SERVICE_PREFIX + "settings.retrySettings.totalTimeout", + datastoreOptions.getRetrySettings().getTotalTimeout().toString()) + .build()); + } + + // Add the memory utilization of the client at the time this trace was collected. + long totalMemory = Runtime.getRuntime().totalMemory(); + long freeMemory = Runtime.getRuntime().freeMemory(); + double memoryUtilization = ((double) (totalMemory - freeMemory)) / totalMemory; + spanBuilder.setAttribute( + ATTRIBUTE_SERVICE_PREFIX + "memoryUtilization", + String.format("%.2f", memoryUtilization * 100) + "%"); + + return spanBuilder; + } + + @Override + public Span startSpan(String spanName) { + SpanBuilder spanBuilder = tracer.spanBuilder(spanName).setSpanKind(SpanKind.PRODUCER); + io.opentelemetry.api.trace.Span span = + addSettingsAttributesToCurrentSpan(spanBuilder).startSpan(); + return new Span(span, spanName); + } + + @Override + public TraceUtil.Span startSpan(String spanName, TraceUtil.Context parent) { + assert (parent instanceof Context); + SpanBuilder spanBuilder = + tracer + .spanBuilder(spanName) + .setSpanKind(SpanKind.PRODUCER) + .setParent(((Context) parent).context); + io.opentelemetry.api.trace.Span span = + addSettingsAttributesToCurrentSpan(spanBuilder).startSpan(); + return new Span(span, spanName); + } + + @Nonnull + @Override + public TraceUtil.Span currentSpan() { + return new Span(io.opentelemetry.api.trace.Span.current(), ""); + } + + @Nonnull + @Override + public TraceUtil.Context currentContext() { + return new Context(io.opentelemetry.context.Context.current()); + } +} diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java new file mode 100644 index 000000000..4e55a1ff6 --- /dev/null +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -0,0 +1,129 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.core.ApiFunction; +import com.google.api.core.ApiFuture; +import com.google.api.core.InternalExtensionOnly; +import com.google.cloud.datastore.DatastoreOptions; +import io.grpc.ManagedChannelBuilder; +import java.util.Map; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** Utility interface to manage OpenTelemetry tracing instrumentation based on the configuration. */ +@InternalExtensionOnly +public interface TraceUtil { + static final String ATTRIBUTE_SERVICE_PREFIX = "gcp.datastore."; + static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; + static final String LIBRARY_NAME = "com.google.cloud.datastore"; + + /** + * Creates and returns an instance of the TraceUtil class. + * + * @param datastoreOptions The DatastoreOptions object that is requesting an instance of + * TraceUtil. + * @return An instance of the TraceUtil class. + */ + static TraceUtil getInstance(@Nonnull DatastoreOptions datastoreOptions) { + boolean createEnabledInstance = datastoreOptions.getOpenTelemetryOptions().isEnabled(); + + // The environment variable can override options to enable/disable telemetry collection. + String enableTracingEnvVar = System.getenv(ENABLE_TRACING_ENV_VAR); + if (enableTracingEnvVar != null) { + if (enableTracingEnvVar.equalsIgnoreCase("true") + || enableTracingEnvVar.equalsIgnoreCase("on")) { + createEnabledInstance = true; + } + if (enableTracingEnvVar.equalsIgnoreCase("false") + || enableTracingEnvVar.equalsIgnoreCase("off")) { + createEnabledInstance = false; + } + } + + if (createEnabledInstance) { + return new EnabledTraceUtil(datastoreOptions); + } else { + return new DisabledTraceUtil(); + } + } + + /** Returns a channel configurator for gRPC, or {@code null} if tracing is disabled. */ + @Nullable + ApiFunction getChannelConfigurator(); + + /** Represents a trace span. */ + interface Span { + /** Adds the given event to this span. */ + Span addEvent(String name); + + /** Adds the given event with the given attributes to this span. */ + Span addEvent(String name, Map attributes); + + /** Adds the given attribute to this span. */ + Span setAttribute(String key, int value); + + /** Adds the given attribute to this span. */ + Span setAttribute(String key, String value); + + /** Marks this span as the current span. */ + Scope makeCurrent(); + + /** Ends this span. */ + void end(); + + /** Ends this span in an error. */ + void end(Throwable error); + + /** + * If an operation ends in the future, its relevant span should end _after_ the future has been + * completed. This method "appends" the span completion code at the completion of the given + * future. In order for telemetry info to be recorded, the future returned by this method should + * be completed. + */ + void endAtFuture(ApiFuture futureValue); + } + + /** Represents a trace context. */ + interface Context { + /** Makes this context the current context. */ + Scope makeCurrent(); + } + + /** Represents a trace scope. */ + interface Scope extends AutoCloseable { + /** Closes the current scope. */ + void close(); + } + + /** Starts a new span with the given name, sets it as the current span, and returns it. */ + Span startSpan(String spanName); + + /** + * Starts a new span with the given name and the given context as its parent, sets it as the + * current span, and returns it. + */ + Span startSpan(String spanName, Context parent); + + /** Returns the current span. */ + @Nonnull + Span currentSpan(); + + /** Returns the current Context. */ + @Nonnull + Context currentContext(); +} diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java index a545580e2..85703f739 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java @@ -70,6 +70,20 @@ public void testHost() { assertEquals("http://localhost:" + PORT, options.build().getHost()); } + @Test + public void testOpenTelemetryOptionsEnabled() { + options.setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(true).build()); + assertTrue(options.build().getOpenTelemetryOptions().isEnabled()); + } + + @Test + public void testOpenTelemetryOptionsDisabled() { + options.setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(false).build()); + assertTrue(!options.build().getOpenTelemetryOptions().isEnabled()); + } + @Test public void testNamespace() { assertTrue(options.build().getNamespace().isEmpty()); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java new file mode 100644 index 000000000..0f3f183cd --- /dev/null +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.datastore.telemetry; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +public class DisabledTraceUtilTest { + @Test + public void disabledTraceUtilDoesNotProvideChannelConfigurator() { + DisabledTraceUtil traceUtil = new DisabledTraceUtil(); + assertThat(traceUtil.getChannelConfigurator()).isNull(); + } + + @Test + public void usesDisabledContext() { + DisabledTraceUtil traceUtil = new DisabledTraceUtil(); + assertThat(traceUtil.currentContext() instanceof DisabledTraceUtil.Context).isTrue(); + } + + @Test + public void usesDisabledSpan() { + DisabledTraceUtil traceUtil = new DisabledTraceUtil(); + assertThat(traceUtil.currentSpan() instanceof DisabledTraceUtil.Span).isTrue(); + assertThat(traceUtil.startSpan("foo") instanceof DisabledTraceUtil.Span).isTrue(); + assertThat( + traceUtil.startSpan("foo", traceUtil.currentContext()) + instanceof DisabledTraceUtil.Span) + .isTrue(); + } + + @Test + public void usesDisabledScope() { + DisabledTraceUtil traceUtil = new DisabledTraceUtil(); + assertThat(traceUtil.currentContext().makeCurrent() instanceof DisabledTraceUtil.Scope) + .isTrue(); + assertThat(traceUtil.currentSpan().makeCurrent() instanceof DisabledTraceUtil.Scope).isTrue(); + } +} diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java new file mode 100644 index 000000000..e88e1a849 --- /dev/null +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java @@ -0,0 +1,104 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.datastore.telemetry; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.NoCredentials; +import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; +import com.google.cloud.datastore.DatastoreOptions; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import org.junit.Before; +import org.junit.Test; + +public class EnabledTraceUtilTest { + @Before + public void setUp() { + GlobalOpenTelemetry.resetForTest(); + } + + DatastoreOptions.Builder getBaseOptions() { + return DatastoreOptions.newBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()); + } + + DatastoreOptions getTracingEnabledOptions() { + return getBaseOptions() + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(true).build()) + .build(); + } + + EnabledTraceUtil newEnabledTraceUtil() { + return new EnabledTraceUtil(getTracingEnabledOptions()); + } + + @Test + public void usesOpenTelemetryFromOptions() { + OpenTelemetrySdk myOpenTelemetrySdk = OpenTelemetrySdk.builder().build(); + DatastoreOptions firestoreOptions = + getBaseOptions() + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder() + .setTracingEnabled(true) + .setOpenTelemetry(myOpenTelemetrySdk) + .build()) + .build(); + EnabledTraceUtil traceUtil = new EnabledTraceUtil(firestoreOptions); + assertThat(traceUtil.getOpenTelemetry()).isEqualTo(myOpenTelemetrySdk); + } + + @Test + public void usesGlobalOpenTelemetryIfOpenTelemetryInstanceNotProvided() { + OpenTelemetrySdk globalOpenTelemetrySdk = OpenTelemetrySdk.builder().buildAndRegisterGlobal(); + DatastoreOptions firestoreOptions = + getBaseOptions() + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(true).build()) + .build(); + EnabledTraceUtil traceUtil = new EnabledTraceUtil(firestoreOptions); + assertThat(traceUtil.getOpenTelemetry()).isEqualTo(GlobalOpenTelemetry.get()); + } + + @Test + public void enabledTraceUtilProvidesChannelConfigurator() { + assertThat(newEnabledTraceUtil().getChannelConfigurator()).isNull(); + } + + @Test + public void usesEnabledContext() { + assertThat(newEnabledTraceUtil().currentContext() instanceof EnabledTraceUtil.Context).isTrue(); + } + + @Test + public void usesEnabledSpan() { + EnabledTraceUtil traceUtil = newEnabledTraceUtil(); + assertThat(traceUtil.currentSpan() instanceof EnabledTraceUtil.Span).isTrue(); + assertThat(traceUtil.startSpan("foo") != null).isTrue(); + assertThat( + traceUtil.startSpan("foo", traceUtil.currentContext()) instanceof EnabledTraceUtil.Span) + .isTrue(); + } + + @Test + public void usesEnabledScope() { + EnabledTraceUtil traceUtil = newEnabledTraceUtil(); + assertThat(traceUtil.currentContext().makeCurrent() instanceof EnabledTraceUtil.Scope).isTrue(); + assertThat(traceUtil.currentSpan().makeCurrent() instanceof EnabledTraceUtil.Scope).isTrue(); + } +} diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/TraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/TraceUtilTest.java new file mode 100644 index 000000000..f1cce8006 --- /dev/null +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/TraceUtilTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.datastore.telemetry; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.NoCredentials; +import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; +import com.google.cloud.datastore.DatastoreOptions; +import org.junit.Test; + +public class TraceUtilTest { + @Test + public void defaultOptionsUseDisabledTraceUtil() { + TraceUtil traceUtil = + TraceUtil.getInstance( + DatastoreOptions.newBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .build()); + assertThat(traceUtil instanceof DisabledTraceUtil).isTrue(); + } + + @Test + public void tracingDisabledOptionsUseDisabledTraceUtil() { + TraceUtil traceUtil = + TraceUtil.getInstance( + DatastoreOptions.newBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(false).build()) + .build()); + assertThat(traceUtil instanceof DisabledTraceUtil).isTrue(); + } + + @Test + public void tracingEnabledOptionsUseEnabledTraceUtil() { + TraceUtil traceUtil = + TraceUtil.getInstance( + DatastoreOptions.newBuilder() + .setProjectId("test-project") + .setCredentials(NoCredentials.getInstance()) + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(true).build()) + .build()); + assertThat(traceUtil instanceof EnabledTraceUtil).isTrue(); + } +} From 77810da52c57bfdb0edd2733df46ad07d956a6c9 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 15 May 2024 14:57:32 -0700 Subject: [PATCH 02/41] feat: Adding Lookup RPC OpenTelemetry Tracing (#1437) * feat: Adding Lookup RPC OpenTelemetry Tracing - Removed OpenCensus Tracing - Added E2E tests with Global and Local OTel SDK - Moved OTel SDK setup to RemoteDatastoreHelper - Fixed pom to depend on BOM for all shared dependencies --- google-cloud-datastore/pom.xml | 58 +- .../google/cloud/datastore/DatastoreImpl.java | 27 +- .../telemetry/DisabledTraceUtil.java | 5 + .../datastore/telemetry/EnabledTraceUtil.java | 6 + .../cloud/datastore/telemetry/TraceUtil.java | 4 + .../testing/RemoteDatastoreHelper.java | 29 +- .../cloud/datastore/it/ITE2ETracingTest.java | 530 ++++++++++++++++++ 7 files changed, 645 insertions(+), 14 deletions(-) create mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index f11be9f56..478c6f785 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -18,6 +18,18 @@ google-cloud-datastore 1.37.0 + + + + + com.google.cloud + gapic-libraries-bom + 1.37.0 + pom + import + + + com.google.api.grpc @@ -117,6 +129,11 @@ + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + io.opentelemetry opentelemetry-api @@ -136,7 +153,6 @@ test-jar test - com.google.guava guava-testlib @@ -178,12 +194,50 @@ 1.4.3 test + + com.google.testparameterinjector + test-parameter-injector + 1.16 + test + + io.opentelemetry - opentelemetry-sdk + opentelemetry-sdk-common + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-trace ${opentelemetry.version} test + + io.opentelemetry + opentelemetry-semconv + 1.1.0-alpha + test + + + + + com.google.cloud.opentelemetry + exporter-trace + 0.15.0 + test + + + com.google.cloud + google-cloud-trace + test + + + com.google.api.grpc + proto-google-cloud-trace-v1 + test + + diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index a3bfb3796..401e3ed54 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.datastore.v1.ExplainOptions; @@ -61,6 +62,9 @@ final class DatastoreImpl extends BaseService implements Datas TransactionOperationExceptionHandler.build(); private final TraceUtil traceUtil = TraceUtil.getInstance(); + private final com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil = + getOptions().getTraceUtil(); + private final ReadOptionProtoPreparer readOptionProtoPreparer; private final AggregationQueryExecutor aggregationQueryExecutor; @@ -450,13 +454,26 @@ protected Entity computeNext() { com.google.datastore.v1.LookupResponse lookup( final com.google.datastore.v1.LookupRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_LOOKUP); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP); + final ReadOptions readOptions = requestPb.getReadOptions(); + span.setAttribute( + "isTransactional", (readOptions.hasTransaction() || readOptions.hasNewTransaction())); + + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( new Callable() { @Override public com.google.datastore.v1.LookupResponse call() throws DatastoreException { - return datastoreRpc.lookup(requestPb); + com.google.datastore.v1.LookupResponse response = datastoreRpc.lookup(requestPb); + span.addEvent( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP + ": Completed", + new ImmutableMap.Builder() + .put("Received", response.getFoundCount()) + .put("Missing", response.getMissingCount()) + .put("Deferred", response.getDeferredCount()) + .build()); + return response; } }, retrySettings, @@ -465,10 +482,10 @@ public com.google.datastore.v1.LookupResponse call() throws DatastoreException { : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java index 21321897d..a4f25813a 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java @@ -61,6 +61,11 @@ public TraceUtil.Span setAttribute(String key, String value) { return this; } + @Override + public TraceUtil.Span setAttribute(String key, boolean value) { + return this; + } + @Override public Scope makeCurrent() { return new Scope(); diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java index 8a5b0b36e..3bf6a7466 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java @@ -175,6 +175,12 @@ public TraceUtil.Span setAttribute(String key, String value) { return this; } + @Override + public TraceUtil.Span setAttribute(String key, boolean value) { + span.setAttribute(ATTRIBUTE_SERVICE_PREFIX + key, value); + return this; + } + @Override public Scope makeCurrent() { try (io.opentelemetry.context.Scope scope = span.makeCurrent()) { diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 4e55a1ff6..c867a5525 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -32,6 +32,7 @@ public interface TraceUtil { static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; + static final String SPAN_NAME_LOOKUP = "Lookup"; /** * Creates and returns an instance of the TraceUtil class. * @@ -80,6 +81,9 @@ interface Span { /** Adds the given attribute to this span. */ Span setAttribute(String key, String value); + /** Adds the given attribute to this span. */ + Span setAttribute(String key, boolean value); + /** Marks this span as the current span. */ Scope makeCurrent(); diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/RemoteDatastoreHelper.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/RemoteDatastoreHelper.java index 596ce96d8..6167cedca 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/RemoteDatastoreHelper.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/testing/RemoteDatastoreHelper.java @@ -19,13 +19,16 @@ import com.google.api.core.InternalApi; import com.google.api.gax.retrying.RetrySettings; import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.http.HttpTransportOptions; +import io.opentelemetry.sdk.OpenTelemetrySdk; import java.util.UUID; +import javax.annotation.Nullable; import org.threeten.bp.Duration; /** @@ -38,13 +41,13 @@ * RetrySettings#getTotalTimeout()} is {@code 120000} and {@link * RetrySettings#getInitialRetryDelay()} is {@code 250}. {@link * HttpTransportOptions#getConnectTimeout()} and {@link HttpTransportOptions#getReadTimeout()} are - * both both set to {@code 60000}. + * both both set to {@code 60000}. If an OpenTelemetrySdk object is passed in, OpenTelemetry Trace + * collection will be enabled for the Client application. * *

Internal testing use only */ @InternalApi public class RemoteDatastoreHelper { - private final DatastoreOptions options; private final Datastore datastore; private final String namespace; @@ -78,18 +81,30 @@ public static RemoteDatastoreHelper create() { } /** Creates a {@code RemoteStorageHelper} object. */ - public static RemoteDatastoreHelper create(String databaseId) { + public static RemoteDatastoreHelper create( + String databaseId, @Nullable OpenTelemetrySdk openTelemetrySdk) { HttpTransportOptions transportOptions = DatastoreOptions.getDefaultHttpTransportOptions(); transportOptions = transportOptions.toBuilder().setConnectTimeout(60000).setReadTimeout(60000).build(); - DatastoreOptions datastoreOption = + DatastoreOptions.Builder datastoreOptionBuilder = DatastoreOptions.newBuilder() .setDatabaseId(databaseId) .setNamespace(UUID.randomUUID().toString()) .setRetrySettings(retrySettings()) - .setTransportOptions(transportOptions) - .build(); - return new RemoteDatastoreHelper(datastoreOption); + .setTransportOptions(transportOptions); + + if (openTelemetrySdk != null) { + datastoreOptionBuilder.setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder() + .setOpenTelemetry(openTelemetrySdk) + .setTracingEnabled(true) + .build()); + } + return new RemoteDatastoreHelper(datastoreOptionBuilder.build()); + } + + public static RemoteDatastoreHelper create(String databaseId) { + return create(databaseId, /*openTelemetrySdk=*/ null); } private static RetrySettings retrySettings() { diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java new file mode 100644 index 000000000..24a9a7149 --- /dev/null +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -0,0 +1,530 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.it; + +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.testing.RemoteDatastoreHelper; +import com.google.cloud.opentelemetry.trace.TraceConfiguration; +import com.google.cloud.opentelemetry.trace.TraceExporter; +import com.google.cloud.trace.v1.TraceServiceClient; +import com.google.common.base.Preconditions; +import com.google.devtools.cloudtrace.v1.Trace; +import com.google.devtools.cloudtrace.v1.TraceSpan; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +// This End-to-End test verifies Client-side Tracing Functionality instrumented using the +// OpenTelemetry API. +// The test depends on the following external APIs/Services: +// 1. Java OpenTelemetry SDK +// 2. Cloud Trace Exporter +// 3. TraceServiceClient from Cloud Trace API v1. +// +// Permissions required to run this test (https://cloud.google.com/trace/docs/iam#trace-roles): +// 1. gcloud auth application-default login must be run with the test user. +// 2. To write traces, test user must have one of roles/cloudtrace.[admin|agent|user] roles. +// 3. To read traces, test user must have one of roles/cloudtrace.[admin|user] roles. +// +// Each test-case has the following workflow: +// 1. OpenTelemetry SDK is initialized with Cloud Trace Exporter and 100% Trace Sampling +// 2. On initialization, Datastore client is provided the OpenTelemetry SDK object from (1) +// 3. A custom TraceID is generated and injected using a custom SpanContext +// 4. Datastore operations are run inside a root TraceSpan created using the custom SpanContext from +// (3). +// 5. Traces are read-back using TraceServiceClient and verified against expected Call Stacks. +@RunWith(TestParameterInjector.class) +public class ITE2ETracingTest { + protected boolean isUsingGlobalOpenTelemetrySDK() { + return useGlobalOpenTelemetrySDK; + } + + protected String datastoreNamedDatabase() { + return datastoreNamedDatabase; + } + + // Helper class to track call-stacks in a trace + protected static class TraceContainer { + + // Maps Span ID to TraceSpan + private final Map idSpanMap; + + // Maps Parent Span ID to a list of Child SpanIDs, useful for top-down traversal + private final Map> parentChildIdMap; + + // Tracks the Root Span ID + private long rootId; + + public TraceContainer(String rootSpanName, Trace trace) { + idSpanMap = new TreeMap<>(); + parentChildIdMap = new TreeMap<>(); + for (TraceSpan span : trace.getSpansList()) { + long spanId = span.getSpanId(); + idSpanMap.put(spanId, span); + if (rootSpanName.equals(span.getName())) { + rootId = span.getSpanId(); + } + + // Add self as a child of the parent span + if (!parentChildIdMap.containsKey(span.getParentSpanId())) { + parentChildIdMap.put(span.getParentSpanId(), new ArrayList<>()); + } + parentChildIdMap.get(span.getParentSpanId()).add(spanId); + } + } + + String spanName(long spanId) { + return idSpanMap.get(spanId).getName(); + } + + List childSpans(long spanId) { + return parentChildIdMap.get(spanId); + } + + // This method only works for matching call stacks with traces which have children of distinct + // type at all levels. This is good enough as the intention is to validate if the e2e path is + // WAI - the intention is not to validate Cloud Trace's correctness w.r.t. durability of all + // kinds of traces. + boolean containsCallStack(String... callStack) throws RuntimeException { + List expectedCallStack = Arrays.asList(callStack); + if (expectedCallStack.isEmpty()) { + throw new RuntimeException("Input callStack is empty"); + } + return dfsContainsCallStack(rootId, expectedCallStack); + } + + // Depth-first check for call stack in the trace + private boolean dfsContainsCallStack(long spanId, List expectedCallStack) { + logger.info( + "span=" + + spanName(spanId) + + ", expectedCallStack[0]=" + + (expectedCallStack.isEmpty() ? "null" : expectedCallStack.get(0))); + if (expectedCallStack.isEmpty()) { + return false; + } + if (spanName(spanId).equals(expectedCallStack.get(0))) { + // Recursion termination + if (childSpans(spanId) == null) { + logger.info("No more children for " + spanName(spanId)); + return expectedCallStack.size() <= 1; + } else { + // Examine the child spans + for (Long childSpan : childSpans(spanId)) { + int callStackListSize = expectedCallStack.size(); + logger.info( + "childSpan=" + + spanName(childSpan) + + ", expectedCallStackSize=" + + callStackListSize); + if (dfsContainsCallStack( + childSpan, + expectedCallStack.subList( + /*fromIndexInclusive=*/ 1, /*toIndexExclusive*/ callStackListSize))) { + return true; + } + } + } + } else { + logger.info(spanName(spanId) + " didn't match " + expectedCallStack.get(0)); + } + return false; + } + } + + private static final Logger logger = Logger.getLogger(ITE2ETracingTest.class.getName()); + + private static final String RUN_AGGREGATION_QUERY_RPC_NAME = "RunAggregationQuery"; + + private static final String RUN_QUERY_RPC_NAME = "RunQuery"; + + private static final int NUM_TRACE_ID_BYTES = 32; + + private static final int NUM_SPAN_ID_BYTES = 16; + + private static final int GET_TRACE_RETRY_COUNT = 60; + + private static final int GET_TRACE_RETRY_BACKOFF_MILLIS = 1000; + + private static final int TRACE_FORCE_FLUSH_MILLIS = 5000; + + private static final int TRACE_PROVIDER_SHUTDOWN_MILLIS = 1000; + + private static Key KEY1; + + // Random int generator for trace ID and span ID + private static Random random; + + private static TraceExporter traceExporter; + + // Required for reading back traces from Cloud Trace for validation + private static TraceServiceClient traceClient_v1; + + // Custom SpanContext for each test, required for TraceID injection + private static SpanContext customSpanContext; + + // Trace read back from Cloud Trace using traceClient_v1 for verification + private static Trace retrievedTrace; + + private static String rootSpanName; + private static Tracer tracer; + + // Required to set custom-root span + private static OpenTelemetrySdk openTelemetrySdk; + + private static String projectId; + + private static DatastoreOptions options; + + private static Datastore datastore; + + @TestParameter boolean useGlobalOpenTelemetrySDK; + + @TestParameter({"default", "test-db"}) + String datastoreNamedDatabase; + + @BeforeClass + public static void setup() throws IOException { + projectId = DatastoreOptions.getDefaultProjectId(); + traceExporter = + TraceExporter.createWithConfiguration( + TraceConfiguration.builder().setProjectId(projectId).build()); + traceClient_v1 = TraceServiceClient.create(); + random = new Random(); + } + + @Before + public void before() throws Exception { + // Set up OTel SDK + Resource resource = + Resource.getDefault().merge(Resource.builder().put(SERVICE_NAME, "Sparky").build()); + + if (isUsingGlobalOpenTelemetrySDK()) { + openTelemetrySdk = + OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(traceExporter).build()) + .setSampler(Sampler.alwaysOn()) + .build()) + .buildAndRegisterGlobal(); + } else { + openTelemetrySdk = + OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(traceExporter).build()) + .setSampler(Sampler.alwaysOn()) + .build()) + .build(); + } + + // Initialize the Datastore DB w/ the OTel SDK. Ideally we'd do this is the @BeforeAll method + // but because gRPC traces need to be deterministically force-flushed for every test + String namedDb = datastoreNamedDatabase(); + logger.log(Level.INFO, "Integration test using named database " + namedDb); + RemoteDatastoreHelper remoteDatastoreHelper = + RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); + options = remoteDatastoreHelper.getOptions(); + datastore = options.getService(); + + Preconditions.checkNotNull( + datastore, + "Error instantiating Datastore. Check that the service account credentials " + + "were properly set."); + + String projectId = options.getProjectId(); + String kind1 = "kind1"; + KEY1 = Key.newBuilder(projectId, kind1, "name", options.getDatabaseId()).build(); + + // Set up the tracer for custom TraceID injection + rootSpanName = + String.format("%s%d", this.getClass().getSimpleName(), System.currentTimeMillis()); + if (isUsingGlobalOpenTelemetrySDK()) { + tracer = GlobalOpenTelemetry.getTracer(rootSpanName); + } else { + tracer = + datastore + .getOptions() + .getOpenTelemetryOptions() + .getOpenTelemetry() + .getTracer(rootSpanName); + } + + // Get up a new SpanContext (ergo TraceId) for each test + customSpanContext = getNewSpanContext(); + assertNotNull(customSpanContext); + assertNull(retrievedTrace); + } + + @After + public void after() throws Exception { + if (isUsingGlobalOpenTelemetrySDK()) { + GlobalOpenTelemetry.resetForTest(); + } + rootSpanName = null; + tracer = null; + retrievedTrace = null; + customSpanContext = null; + } + + @AfterClass + public static void teardown() throws Exception { + traceClient_v1.close(); + CompletableResultCode completableResultCode = + openTelemetrySdk.getSdkTracerProvider().shutdown(); + completableResultCode.join(TRACE_PROVIDER_SHUTDOWN_MILLIS, TimeUnit.MILLISECONDS); + } + + // Generates a random hex string of length `numBytes` + private String generateRandomHexString(int numBytes) { + StringBuilder newTraceId = new StringBuilder(); + while (newTraceId.length() < numBytes) { + newTraceId.append(Integer.toHexString(random.nextInt())); + } + return newTraceId.substring(0, numBytes); + } + + protected String generateNewTraceId() { + return generateRandomHexString(NUM_TRACE_ID_BYTES); + } + + // Generates a random 16-byte hex string + protected String generateNewSpanId() { + return generateRandomHexString(NUM_SPAN_ID_BYTES); + } + + // Generates a new SpanContext w/ random traceId,spanId + protected SpanContext getNewSpanContext() { + String traceId = generateNewTraceId(); + String spanId = generateNewSpanId(); + logger.info("traceId=" + traceId + ", spanId=" + spanId); + + return SpanContext.create(traceId, spanId, TraceFlags.getSampled(), TraceState.getDefault()); + } + + protected Span getNewRootSpanWithContext() { + // Execute the DB operation in the context of the custom root span. + return tracer + .spanBuilder(rootSpanName) + .setParent(Context.root().with(Span.wrap(customSpanContext))) + .startSpan(); + } + + protected void waitForTracesToComplete() throws Exception { + logger.info("Flushing traces..."); + CompletableResultCode completableResultCode = + openTelemetrySdk.getSdkTracerProvider().forceFlush(); + completableResultCode.join(TRACE_FORCE_FLUSH_MILLIS, TimeUnit.MILLISECONDS); + } + + // Validates `retrievedTrace`. Cloud Trace indexes traces w/ eventual consistency, even when + // indexing traceId, therefore the test must retry a few times before the complete trace is + // available. + // For Transaction traces, there may be more spans than in the trace than specified in + // `callStack`. So `numExpectedSpans` is the expected total number of spans (and not just the + // spans in `callStack`) + protected void fetchAndValidateTrace( + String traceId, int numExpectedSpans, List> callStackList) + throws InterruptedException { + // Large enough count to accommodate eventually consistent Cloud Trace backend + int numRetries = GET_TRACE_RETRY_COUNT; + // Account for rootSpanName + numExpectedSpans++; + + // Fetch traces + do { + try { + retrievedTrace = traceClient_v1.getTrace(projectId, traceId); + assertEquals(traceId, retrievedTrace.getTraceId()); + + logger.info( + "expectedSpanCount=" + + numExpectedSpans + + ", retrievedSpanCount=" + + retrievedTrace.getSpansCount()); + } catch (NotFoundException notFound) { + logger.info("Trace not found, retrying in " + GET_TRACE_RETRY_BACKOFF_MILLIS + " ms"); + } catch (IndexOutOfBoundsException outOfBoundsException) { + logger.info("Call stack not found in trace. Retrying."); + } + if (retrievedTrace == null || numExpectedSpans != retrievedTrace.getSpansCount()) { + Thread.sleep(GET_TRACE_RETRY_BACKOFF_MILLIS); + } + } while (numRetries-- > 0 + && (retrievedTrace == null || numExpectedSpans != retrievedTrace.getSpansCount())); + + if (retrievedTrace == null || numExpectedSpans != retrievedTrace.getSpansCount()) { + throw new RuntimeException( + "Expected number of spans: " + + numExpectedSpans + + ", Actual number of spans: " + + (retrievedTrace != null + ? retrievedTrace.getSpansList().toString() + : "Trace NOT_FOUND")); + } + + TraceContainer traceContainer = new TraceContainer(rootSpanName, retrievedTrace); + + for (List callStack : callStackList) { + // Update all call stacks to be rooted at rootSpanName + ArrayList expectedCallStack = new ArrayList<>(callStack); + + // numExpectedSpans should account for rootSpanName (not passed in callStackList) + expectedCallStack.add(0, rootSpanName); + + // *May be* the full trace was returned + logger.info("Checking if TraceContainer contains the callStack"); + String[] expectedCallList = new String[expectedCallStack.size()]; + if (!traceContainer.containsCallStack(expectedCallStack.toArray(expectedCallList))) { + throw new RuntimeException( + "Expected spans: " + + Arrays.toString(expectedCallList) + + ", Actual spans: " + + (retrievedTrace != null + ? retrievedTrace.getSpansList().toString() + : "Trace NOT_FOUND")); + } + logger.severe("CallStack not found in TraceContainer."); + } + } + + // Validates `retrievedTrace`. Cloud Trace indexes traces w/ eventual consistency, even when + // indexing traceId, therefore the test must retry a few times before the complete trace is + // available. + // For Non-Transaction traces, there is a 1:1 ratio of spans in `spanNames` and in the trace. + protected void fetchAndValidateTrace(String traceId, String... spanNames) + throws InterruptedException { + fetchAndValidateTrace(traceId, spanNames.length, Arrays.asList(Arrays.asList(spanNames))); + } + + @Test + public void traceContainerTest() throws Exception { + // Make sure the test has a new SpanContext (and TraceId for injection) + assertNotNull(customSpanContext); + + // Inject new trace ID + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + Entity entity = datastore.get(KEY1); + assertNull(entity); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + Trace traceResp = null; + int expectedSpanCount = 2; + + int numRetries = GET_TRACE_RETRY_COUNT; + do { + try { + traceResp = traceClient_v1.getTrace(projectId, customSpanContext.getTraceId()); + if (traceResp.getSpansCount() == expectedSpanCount) { + logger.info("Success: Got " + expectedSpanCount + " spans."); + break; + } + } catch (NotFoundException notFoundException) { + Thread.sleep(GET_TRACE_RETRY_BACKOFF_MILLIS); + logger.info("Trace not found, retrying in " + GET_TRACE_RETRY_BACKOFF_MILLIS + " ms"); + } + logger.info( + "Trace Found. The trace did not contain " + + expectedSpanCount + + " spans. Going to retry."); + numRetries--; + } while (numRetries > 0); + + // Make sure we got as many spans as we expected. + assertNotNull(traceResp); + assertEquals(expectedSpanCount, traceResp.getSpansCount()); + + TraceContainer traceCont = new TraceContainer(rootSpanName, traceResp); + + // Contains exact path + assertTrue(traceCont.containsCallStack(rootSpanName, SPAN_NAME_LOOKUP)); + + // Top-level mismatch + assertFalse(traceCont.containsCallStack(SPAN_NAME_LOOKUP, RUN_QUERY_RPC_NAME)); + + // Leaf-level mismatch/missing + assertFalse( + traceCont.containsCallStack( + rootSpanName, SPAN_NAME_LOOKUP, RUN_AGGREGATION_QUERY_RPC_NAME)); + } + + @Test + public void lookupTraceTest() throws Exception { + // Make sure the test has a new SpanContext (and TraceId for injection) + assertNotNull(customSpanContext); + + // Inject new trace ID + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + Entity entity = datastore.get(KEY1); + assertNull(entity); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_LOOKUP); + } +} From 730aa427d8562ce3906fb80c3809cb5a27af0ac2 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 20 May 2024 11:13:55 -0700 Subject: [PATCH 03/41] feat: Adding Commit RPC Trace Instrumentation (#1440) - Added end-to-end test for Datastore operationsput, add, update and delete. - Updated E2E Test to use the namespace correctly for efficient clean-up of test data --- .../google/cloud/datastore/DatastoreImpl.java | 11 +- .../cloud/datastore/telemetry/TraceUtil.java | 3 + .../cloud/datastore/it/ITE2ETracingTest.java | 109 +++++++++++++++++- 3 files changed, 115 insertions(+), 8 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 401e3ed54..cb60685c7 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -619,8 +619,11 @@ private com.google.datastore.v1.CommitResponse commitMutation( com.google.datastore.v1.CommitResponse commit( final com.google.datastore.v1.CommitRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_COMMIT); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT); + span.setAttribute("isTransactional", requestPb.hasTransaction()); + + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.commit(requestPb), retrySettings, @@ -629,10 +632,10 @@ com.google.datastore.v1.CommitResponse commit( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index c867a5525..fe4effb1d 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -33,6 +33,9 @@ public interface TraceUtil { static final String LIBRARY_NAME = "com.google.cloud.datastore"; static final String SPAN_NAME_LOOKUP = "Lookup"; + + static final String SPAN_NAME_COMMIT = "Commit"; + /** * Creates and returns an instance of the TraceUtil class. * diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 24a9a7149..2fb139289 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -16,6 +16,7 @@ package com.google.cloud.datastore.it; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; @@ -207,6 +208,8 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY1; + private static Key KEY2; + // Random int generator for trace ID and span ID private static Random random; @@ -233,9 +236,15 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Datastore datastore; + private static RemoteDatastoreHelper remoteDatastoreHelper; + @TestParameter boolean useGlobalOpenTelemetrySDK; - @TestParameter({"default", "test-db"}) + @TestParameter({ + /*(default)*/ + "", + "test-db" + }) String datastoreNamedDatabase; @BeforeClass @@ -280,8 +289,7 @@ public void before() throws Exception { // but because gRPC traces need to be deterministically force-flushed for every test String namedDb = datastoreNamedDatabase(); logger.log(Level.INFO, "Integration test using named database " + namedDb); - RemoteDatastoreHelper remoteDatastoreHelper = - RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); + remoteDatastoreHelper = RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); options = remoteDatastoreHelper.getOptions(); datastore = options.getService(); @@ -292,7 +300,14 @@ public void before() throws Exception { String projectId = options.getProjectId(); String kind1 = "kind1"; - KEY1 = Key.newBuilder(projectId, kind1, "name", options.getDatabaseId()).build(); + KEY1 = + Key.newBuilder(projectId, kind1, "name1", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY2 = + Key.newBuilder(projectId, kind1, "name2", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); // Set up the tracer for custom TraceID injection rootSpanName = @@ -319,6 +334,7 @@ public void after() throws Exception { if (isUsingGlobalOpenTelemetrySDK()) { GlobalOpenTelemetry.resetForTest(); } + remoteDatastoreHelper.deleteNamespace(); rootSpanName = null; tracer = null; retrievedTrace = null; @@ -527,4 +543,89 @@ public void lookupTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_LOOKUP); } + + @Test + public void commitTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + try (Scope ignored = rootSpan.makeCurrent()) { + Entity response = datastore.add(entity1); + assertEquals(entity1, response); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } + + @Test + public void putTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + try (Scope ignored = rootSpan.makeCurrent()) { + Entity response = datastore.put(entity1); + assertEquals(entity1, response); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } + + @Test + public void updateTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + Span rootSpan = getNewRootSpanWithContext(); + + try (Scope ignored = rootSpan.makeCurrent()) { + Entity entity1_update = + Entity.newBuilder(entity1).set("test_field", "new_test_value1").build(); + Entity entity2_update = + Entity.newBuilder(entity2).set("test_field", "new_test_value1").build(); + datastore.update(entity1_update, entity2_update); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } + + @Test + public void deleteTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + Entity response = datastore.put(entity1); + assertEquals(entity1, response); + + Span rootSpan = getNewRootSpanWithContext(); + + try (Scope ignored = rootSpan.makeCurrent()) { + datastore.delete(entity1.getKey()); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } } From 602c5e314106704eed068375df405296bef74deb Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 23 May 2024 10:15:00 -0700 Subject: [PATCH 04/41] feat: RunQuery trace instrumentation (#1441) * feat: RunQuery trace instrumentation --- .../google/cloud/datastore/DatastoreImpl.java | 37 ++++++++++++------ .../cloud/datastore/telemetry/TraceUtil.java | 3 +- .../cloud/datastore/it/ITE2ETracingTest.java | 38 +++++++++++++++++-- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index cb60685c7..ebf5aa7e9 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -35,6 +35,7 @@ import com.google.datastore.v1.ExplainOptions; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.ReserveIdsRequest; +import com.google.datastore.v1.RunQueryResponse; import com.google.datastore.v1.TransactionOptions; import com.google.protobuf.ByteString; import io.opencensus.common.Scope; @@ -240,20 +241,34 @@ public AggregationResults runAggregation( com.google.datastore.v1.RunQueryResponse runQuery( final com.google.datastore.v1.RunQueryRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_RUNQUERY); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { - return RetryHelper.runWithRetries( - () -> datastoreRpc.runQuery(requestPb), - retrySettings, - requestPb.getReadOptions().getTransaction().isEmpty() - ? EXCEPTION_HANDLER - : TRANSACTION_OPERATION_EXCEPTION_HANDLER, - getOptions().getClock()); + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY); + ReadOptions readOptions = requestPb.getReadOptions(); + span.setAttribute( + "isTransactional", readOptions.hasTransaction() || readOptions.hasNewTransaction()); + span.setAttribute("readConsistency", readOptions.getReadConsistency().toString()); + + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + RunQueryResponse response = + RetryHelper.runWithRetries( + () -> datastoreRpc.runQuery(requestPb), + retrySettings, + requestPb.getReadOptions().getTransaction().isEmpty() + ? EXCEPTION_HANDLER + : TRANSACTION_OPERATION_EXCEPTION_HANDLER, + getOptions().getClock()); + span.addEvent( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY + ": Completed", + new ImmutableMap.Builder() + .put("Received", response.getBatch().getEntityResultsCount()) + .put("More results", response.getBatch().getMoreResults().toString()) + .build()); + return response; } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index fe4effb1d..ccb5cfe7a 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -31,10 +31,9 @@ public interface TraceUtil { static final String ATTRIBUTE_SERVICE_PREFIX = "gcp.datastore."; static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; - static final String SPAN_NAME_LOOKUP = "Lookup"; - static final String SPAN_NAME_COMMIT = "Commit"; + static final String SPAN_NAME_RUN_QUERY = "RunQuery"; /** * Creates and returns an instance of the TraceUtil class. diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 2fb139289..0212b5e42 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -18,6 +18,7 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -30,6 +31,9 @@ import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; import com.google.cloud.opentelemetry.trace.TraceExporter; @@ -301,11 +305,11 @@ public void before() throws Exception { String projectId = options.getProjectId(); String kind1 = "kind1"; KEY1 = - Key.newBuilder(projectId, kind1, "name1", options.getDatabaseId()) + Key.newBuilder(projectId, kind1, "key1", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); KEY2 = - Key.newBuilder(projectId, kind1, "name2", options.getDatabaseId()) + Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); @@ -594,7 +598,6 @@ public void updateTraceTest() throws Exception { assertEquals(entityList, response); Span rootSpan = getNewRootSpanWithContext(); - try (Scope ignored = rootSpan.makeCurrent()) { Entity entity1_update = Entity.newBuilder(entity1).set("test_field", "new_test_value1").build(); @@ -625,7 +628,34 @@ public void deleteTraceTest() throws Exception { rootSpan.end(); } waitForTracesToComplete(); - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); } + + @Test + public void runQueryTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); + } } From 1e4e7ca13a5663681e042f04fd3d4641680f32b6 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 23 May 2024 10:46:31 -0700 Subject: [PATCH 05/41] feat: RunAggregationQuery instrumentation (#1447) * feat: RunQuery trace instrumentation * Formatting * Formatting * Refactor: s/RUNQUERY/RUN_QUERY * feat: RunAggregationQuery Trace Instrumentation * Build: retiring test assertions for OpenCensus spans - will be replacing this in hermetic integration tests for OpenTelemetry using in-memory span exports (in addition to ITE2ETraceTest.java). * Formatting * Fixing @Test annotation missed after merge * Formatting --- .../RetryAndTraceDatastoreRpcDecorator.java | 19 ++--- .../cloud/datastore/telemetry/TraceUtil.java | 1 + ...etryAndTraceDatastoreRpcDecoratorTest.java | 14 +--- .../cloud/datastore/it/ITE2ETracingTest.java | 79 ++++++++++++++++++- 4 files changed, 89 insertions(+), 24 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java index c4a85caab..ba4adb5ff 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java @@ -16,7 +16,6 @@ package com.google.cloud.datastore; import static com.google.cloud.BaseService.EXCEPTION_HANDLER; -import static com.google.cloud.datastore.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import com.google.api.core.InternalApi; import com.google.api.gax.retrying.RetrySettings; @@ -39,9 +38,6 @@ import com.google.datastore.v1.RunAggregationQueryResponse; import com.google.datastore.v1.RunQueryRequest; import com.google.datastore.v1.RunQueryResponse; -import io.opencensus.common.Scope; -import io.opencensus.trace.Span; -import io.opencensus.trace.Status; import java.util.concurrent.Callable; /** @@ -52,7 +48,7 @@ public class RetryAndTraceDatastoreRpcDecorator implements DatastoreRpc { private final DatastoreRpc datastoreRpc; - private final TraceUtil traceUtil; + private final com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil; private final RetrySettings retrySettings; private final DatastoreOptions datastoreOptions; @@ -62,9 +58,9 @@ public RetryAndTraceDatastoreRpcDecorator( RetrySettings retrySettings, DatastoreOptions datastoreOptions) { this.datastoreRpc = datastoreRpc; - this.traceUtil = traceUtil; this.retrySettings = retrySettings; this.datastoreOptions = datastoreOptions; + this.otelTraceUtil = datastoreOptions.getTraceUtil(); } @Override @@ -106,19 +102,20 @@ public RunQueryResponse runQuery(RunQueryRequest request) { @Override public RunAggregationQueryResponse runAggregationQuery(RunAggregationQueryRequest request) { return invokeRpc( - () -> datastoreRpc.runAggregationQuery(request), SPAN_NAME_RUN_AGGREGATION_QUERY); + () -> datastoreRpc.runAggregationQuery(request), + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY); } public O invokeRpc(Callable block, String startSpan) { - Span span = traceUtil.startSpan(startSpan); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(startSpan); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( block, this.retrySettings, EXCEPTION_HANDLER, this.datastoreOptions.getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index ccb5cfe7a..00e9f2376 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -34,6 +34,7 @@ public interface TraceUtil { static final String SPAN_NAME_LOOKUP = "Lookup"; static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; + static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; /** * Creates and returns an instance of the TraceUtil class. diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java index b86355afa..b01bcab82 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java @@ -15,11 +15,8 @@ */ package com.google.cloud.datastore; -import static com.google.cloud.datastore.TraceUtil.END_SPAN_OPTIONS; -import static com.google.cloud.datastore.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.common.truth.Truth.assertThat; import static com.google.rpc.Code.UNAVAILABLE; -import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.createStrictMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; @@ -29,8 +26,6 @@ import com.google.cloud.datastore.spi.v1.DatastoreRpc; import com.google.datastore.v1.RunAggregationQueryRequest; import com.google.datastore.v1.RunAggregationQueryResponse; -import io.opencensus.trace.Span; -import io.opencensus.trace.Tracer; import org.junit.Before; import org.junit.Test; @@ -49,7 +44,6 @@ public class RetryAndTraceDatastoreRpcDecoratorTest { @Before public void setUp() throws Exception { mockDatastoreRpc = createStrictMock(DatastoreRpc.class); - mockTraceUtil = createStrictMock(TraceUtil.class); datastoreRpcDecorator = new RetryAndTraceDatastoreRpcDecorator( mockDatastoreRpc, mockTraceUtil, retrySettings, datastoreOptions); @@ -57,7 +51,6 @@ public void setUp() throws Exception { @Test public void testRunAggregationQuery() { - Span mockSpan = createStrictMock(Span.class); RunAggregationQueryRequest aggregationQueryRequest = RunAggregationQueryRequest.getDefaultInstance(); RunAggregationQueryResponse aggregationQueryResponse = @@ -69,16 +62,13 @@ public void testRunAggregationQuery() { UNAVAILABLE.getNumber(), "API not accessible currently", UNAVAILABLE.name())) .times(2) .andReturn(aggregationQueryResponse); - expect(mockTraceUtil.startSpan(SPAN_NAME_RUN_AGGREGATION_QUERY)).andReturn(mockSpan); - expect(mockTraceUtil.getTracer()).andReturn(createNiceMock(Tracer.class)); - mockSpan.end(END_SPAN_OPTIONS); - replay(mockDatastoreRpc, mockTraceUtil, mockSpan); + replay(mockDatastoreRpc); RunAggregationQueryResponse actualAggregationQueryResponse = datastoreRpcDecorator.runAggregationQuery(aggregationQueryRequest); assertThat(actualAggregationQueryResponse).isSameInstanceAs(aggregationQueryResponse); - verify(mockDatastoreRpc, mockTraceUtil, mockSpan); + verify(mockDatastoreRpc); } } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 0212b5e42..357e748f1 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -16,9 +16,12 @@ package com.google.cloud.datastore.it; +import static com.google.cloud.datastore.aggregation.Aggregation.count; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; +import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -27,12 +30,16 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.datastore.AggregationQuery; +import com.google.cloud.datastore.AggregationResult; +import com.google.cloud.datastore.AggregationResults; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; @@ -95,6 +102,7 @@ // 5. Traces are read-back using TraceServiceClient and verified against expected Call Stacks. @RunWith(TestParameterInjector.class) public class ITE2ETracingTest { + protected boolean isUsingGlobalOpenTelemetrySDK() { return useGlobalOpenTelemetrySDK; } @@ -214,6 +222,10 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY2; + private static Key KEY3; + + private static Key KEY4; + // Random int generator for trace ID and span ID private static Random random; @@ -309,10 +321,17 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY2 = + Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY3 = + Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY4 = Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); - // Set up the tracer for custom TraceID injection rootSpanName = String.format("%s%d", this.getClass().getSimpleName(), System.currentTimeMillis()); @@ -658,4 +677,62 @@ public void runQueryTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); } + + @Test + public void runAggregationQueryTraceTest() throws Exception { + Entity entity1 = + Entity.newBuilder(KEY1) + .set("pepper_name", "jalapeno") + .set("max_scoville_level", 10000) + .build(); + Entity entity2 = + Entity.newBuilder(KEY2) + .set("pepper_name", "serrano") + .set("max_scoville_level", 25000) + .build(); + Entity entity3 = + Entity.newBuilder(KEY3) + .set("pepper_name", "habanero") + .set("max_scoville_level", 350000) + .build(); + Entity entity4 = + Entity.newBuilder(KEY4) + .set("pepper_name", "ghost") + .set("max_scoville_level", 1500000) + .build(); + + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + entityList.add(entity3); + entityList.add(entity4); + + List response = datastore.add(entity1, entity2, entity3, entity4); + assertEquals(entityList, response); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter mediumSpicyFilters = PropertyFilter.lt("max_scoville_level", 100000); + StructuredQuery mediumSpicyQuery = + Query.newEntityQueryBuilder() + .setKind(KEY1.getKind()) + .setFilter(mediumSpicyFilters) + .build(); + AggregationQuery countSpicyPeppers = + Query.newAggregationQueryBuilder() + .addAggregation(count().as("count")) + .over(mediumSpicyQuery) + .build(); + AggregationResults results = datastore.runAggregation(countSpicyPeppers); + assertThat(results.size()).isEqualTo(1); + AggregationResult result = results.get(0); + assertThat(result.getLong("count")).isEqualTo(2L); + } finally { + rootSpan.end(); + } + + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_AGGREGATION_QUERY); + } } From 52329e43dd0e891141d8a1d2e630a18b9005d045 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 15 May 2024 16:25:18 -0700 Subject: [PATCH 06/41] feat: RunQuery trace instrumentation --- .../google/cloud/datastore/DatastoreImpl.java | 48 ++-- .../cloud/datastore/telemetry/TraceUtil.java | 5 +- .../cloud/datastore/it/ITE2ETracingTest.java | 218 +----------------- 3 files changed, 21 insertions(+), 250 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index ebf5aa7e9..401e3ed54 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -35,7 +35,6 @@ import com.google.datastore.v1.ExplainOptions; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.ReserveIdsRequest; -import com.google.datastore.v1.RunQueryResponse; import com.google.datastore.v1.TransactionOptions; import com.google.protobuf.ByteString; import io.opencensus.common.Scope; @@ -241,34 +240,20 @@ public AggregationResults runAggregation( com.google.datastore.v1.RunQueryResponse runQuery( final com.google.datastore.v1.RunQueryRequest requestPb) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY); - ReadOptions readOptions = requestPb.getReadOptions(); - span.setAttribute( - "isTransactional", readOptions.hasTransaction() || readOptions.hasNewTransaction()); - span.setAttribute("readConsistency", readOptions.getReadConsistency().toString()); - - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { - RunQueryResponse response = - RetryHelper.runWithRetries( - () -> datastoreRpc.runQuery(requestPb), - retrySettings, - requestPb.getReadOptions().getTransaction().isEmpty() - ? EXCEPTION_HANDLER - : TRANSACTION_OPERATION_EXCEPTION_HANDLER, - getOptions().getClock()); - span.addEvent( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY + ": Completed", - new ImmutableMap.Builder() - .put("Received", response.getBatch().getEntityResultsCount()) - .put("More results", response.getBatch().getMoreResults().toString()) - .build()); - return response; + Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_RUNQUERY); + try (Scope scope = traceUtil.getTracer().withSpan(span)) { + return RetryHelper.runWithRetries( + () -> datastoreRpc.runQuery(requestPb), + retrySettings, + requestPb.getReadOptions().getTransaction().isEmpty() + ? EXCEPTION_HANDLER + : TRANSACTION_OPERATION_EXCEPTION_HANDLER, + getOptions().getClock()); } catch (RetryHelperException e) { - span.end(e); + span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); throw DatastoreException.translateAndThrow(e); } finally { - span.end(); + span.end(TraceUtil.END_SPAN_OPTIONS); } } @@ -634,11 +619,8 @@ private com.google.datastore.v1.CommitResponse commitMutation( com.google.datastore.v1.CommitResponse commit( final com.google.datastore.v1.CommitRequest requestPb) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT); - span.setAttribute("isTransactional", requestPb.hasTransaction()); - - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_COMMIT); + try (Scope scope = traceUtil.getTracer().withSpan(span)) { return RetryHelper.runWithRetries( () -> datastoreRpc.commit(requestPb), retrySettings, @@ -647,10 +629,10 @@ com.google.datastore.v1.CommitResponse commit( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.end(e); + span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); throw DatastoreException.translateAndThrow(e); } finally { - span.end(); + span.end(TraceUtil.END_SPAN_OPTIONS); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 00e9f2376..c867a5525 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -31,11 +31,8 @@ public interface TraceUtil { static final String ATTRIBUTE_SERVICE_PREFIX = "gcp.datastore."; static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; - static final String SPAN_NAME_LOOKUP = "Lookup"; - static final String SPAN_NAME_COMMIT = "Commit"; - static final String SPAN_NAME_RUN_QUERY = "RunQuery"; - static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; + static final String SPAN_NAME_LOOKUP = "Lookup"; /** * Creates and returns an instance of the TraceUtil class. * diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 357e748f1..24a9a7149 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -16,12 +16,7 @@ package com.google.cloud.datastore.it; -import static com.google.cloud.datastore.aggregation.Aggregation.count; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; -import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -30,17 +25,10 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.rpc.NotFoundException; -import com.google.cloud.datastore.AggregationQuery; -import com.google.cloud.datastore.AggregationResult; -import com.google.cloud.datastore.AggregationResults; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; -import com.google.cloud.datastore.Query; -import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.StructuredQuery; -import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; import com.google.cloud.opentelemetry.trace.TraceExporter; @@ -102,7 +90,6 @@ // 5. Traces are read-back using TraceServiceClient and verified against expected Call Stacks. @RunWith(TestParameterInjector.class) public class ITE2ETracingTest { - protected boolean isUsingGlobalOpenTelemetrySDK() { return useGlobalOpenTelemetrySDK; } @@ -220,12 +207,6 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY1; - private static Key KEY2; - - private static Key KEY3; - - private static Key KEY4; - // Random int generator for trace ID and span ID private static Random random; @@ -252,15 +233,9 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Datastore datastore; - private static RemoteDatastoreHelper remoteDatastoreHelper; - @TestParameter boolean useGlobalOpenTelemetrySDK; - @TestParameter({ - /*(default)*/ - "", - "test-db" - }) + @TestParameter({"default", "test-db"}) String datastoreNamedDatabase; @BeforeClass @@ -305,7 +280,8 @@ public void before() throws Exception { // but because gRPC traces need to be deterministically force-flushed for every test String namedDb = datastoreNamedDatabase(); logger.log(Level.INFO, "Integration test using named database " + namedDb); - remoteDatastoreHelper = RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); + RemoteDatastoreHelper remoteDatastoreHelper = + RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); options = remoteDatastoreHelper.getOptions(); datastore = options.getService(); @@ -316,22 +292,8 @@ public void before() throws Exception { String projectId = options.getProjectId(); String kind1 = "kind1"; - KEY1 = - Key.newBuilder(projectId, kind1, "key1", options.getDatabaseId()) - .setNamespace(options.getNamespace()) - .build(); - KEY2 = - Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) - .setNamespace(options.getNamespace()) - .build(); - KEY3 = - Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) - .setNamespace(options.getNamespace()) - .build(); - KEY4 = - Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) - .setNamespace(options.getNamespace()) - .build(); + KEY1 = Key.newBuilder(projectId, kind1, "name", options.getDatabaseId()).build(); + // Set up the tracer for custom TraceID injection rootSpanName = String.format("%s%d", this.getClass().getSimpleName(), System.currentTimeMillis()); @@ -357,7 +319,6 @@ public void after() throws Exception { if (isUsingGlobalOpenTelemetrySDK()) { GlobalOpenTelemetry.resetForTest(); } - remoteDatastoreHelper.deleteNamespace(); rootSpanName = null; tracer = null; retrievedTrace = null; @@ -566,173 +527,4 @@ public void lookupTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_LOOKUP); } - - @Test - public void commitTraceTest() throws Exception { - assertNotNull(customSpanContext); - - Span rootSpan = getNewRootSpanWithContext(); - - Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); - try (Scope ignored = rootSpan.makeCurrent()) { - Entity response = datastore.add(entity1); - assertEquals(entity1, response); - } finally { - rootSpan.end(); - } - waitForTracesToComplete(); - - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); - } - - @Test - public void putTraceTest() throws Exception { - assertNotNull(customSpanContext); - - Span rootSpan = getNewRootSpanWithContext(); - - Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); - try (Scope ignored = rootSpan.makeCurrent()) { - Entity response = datastore.put(entity1); - assertEquals(entity1, response); - } finally { - rootSpan.end(); - } - waitForTracesToComplete(); - - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); - } - - @Test - public void updateTraceTest() throws Exception { - assertNotNull(customSpanContext); - - Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); - Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); - List entityList = new ArrayList<>(); - entityList.add(entity1); - entityList.add(entity2); - - List response = datastore.add(entity1, entity2); - assertEquals(entityList, response); - - Span rootSpan = getNewRootSpanWithContext(); - try (Scope ignored = rootSpan.makeCurrent()) { - Entity entity1_update = - Entity.newBuilder(entity1).set("test_field", "new_test_value1").build(); - Entity entity2_update = - Entity.newBuilder(entity2).set("test_field", "new_test_value1").build(); - datastore.update(entity1_update, entity2_update); - } finally { - rootSpan.end(); - } - waitForTracesToComplete(); - - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); - } - - @Test - public void deleteTraceTest() throws Exception { - assertNotNull(customSpanContext); - - Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); - Entity response = datastore.put(entity1); - assertEquals(entity1, response); - - Span rootSpan = getNewRootSpanWithContext(); - - try (Scope ignored = rootSpan.makeCurrent()) { - datastore.delete(entity1.getKey()); - } finally { - rootSpan.end(); - } - waitForTracesToComplete(); - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); - } - - @Test - public void runQueryTraceTest() throws Exception { - Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); - Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); - List entityList = new ArrayList<>(); - entityList.add(entity1); - entityList.add(entity2); - - List response = datastore.add(entity1, entity2); - assertEquals(entityList, response); - - Span rootSpan = getNewRootSpanWithContext(); - try (Scope ignored = rootSpan.makeCurrent()) { - PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); - Query query = - Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); - QueryResults queryResults = datastore.run(query); - assertTrue(queryResults.hasNext()); - assertEquals(entity1, queryResults.next()); - assertFalse(queryResults.hasNext()); - } finally { - rootSpan.end(); - } - waitForTracesToComplete(); - - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); - } - - @Test - public void runAggregationQueryTraceTest() throws Exception { - Entity entity1 = - Entity.newBuilder(KEY1) - .set("pepper_name", "jalapeno") - .set("max_scoville_level", 10000) - .build(); - Entity entity2 = - Entity.newBuilder(KEY2) - .set("pepper_name", "serrano") - .set("max_scoville_level", 25000) - .build(); - Entity entity3 = - Entity.newBuilder(KEY3) - .set("pepper_name", "habanero") - .set("max_scoville_level", 350000) - .build(); - Entity entity4 = - Entity.newBuilder(KEY4) - .set("pepper_name", "ghost") - .set("max_scoville_level", 1500000) - .build(); - - List entityList = new ArrayList<>(); - entityList.add(entity1); - entityList.add(entity2); - entityList.add(entity3); - entityList.add(entity4); - - List response = datastore.add(entity1, entity2, entity3, entity4); - assertEquals(entityList, response); - - Span rootSpan = getNewRootSpanWithContext(); - try (Scope ignored = rootSpan.makeCurrent()) { - PropertyFilter mediumSpicyFilters = PropertyFilter.lt("max_scoville_level", 100000); - StructuredQuery mediumSpicyQuery = - Query.newEntityQueryBuilder() - .setKind(KEY1.getKind()) - .setFilter(mediumSpicyFilters) - .build(); - AggregationQuery countSpicyPeppers = - Query.newAggregationQueryBuilder() - .addAggregation(count().as("count")) - .over(mediumSpicyQuery) - .build(); - AggregationResults results = datastore.runAggregation(countSpicyPeppers); - assertThat(results.size()).isEqualTo(1); - AggregationResult result = results.get(0); - assertThat(result.getLong("count")).isEqualTo(2L); - } finally { - rootSpan.end(); - } - - waitForTracesToComplete(); - - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_AGGREGATION_QUERY); - } } From de05e49068982f62c9c6ec4e5a9cff12ce6fb0be Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 15 May 2024 16:38:29 -0700 Subject: [PATCH 07/41] Formatting --- .../java/com/google/cloud/datastore/telemetry/TraceUtil.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index c867a5525..14d8801fc 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -33,6 +33,9 @@ public interface TraceUtil { static final String LIBRARY_NAME = "com.google.cloud.datastore"; static final String SPAN_NAME_LOOKUP = "Lookup"; + + static final String SPAN_NAME_RUNQUERY = "RunQuery"; + /** * Creates and returns an instance of the TraceUtil class. * From a9965db744b0ebb9221ef3a8147e088c1e47349a Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 15 May 2024 16:39:55 -0700 Subject: [PATCH 08/41] Formatting --- .../cloud/datastore/it/ITE2ETracingTest.java | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 24a9a7149..d498d38ba 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -17,6 +17,7 @@ package com.google.cloud.datastore.it; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUNQUERY; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -29,6 +30,9 @@ import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; import com.google.cloud.opentelemetry.trace.TraceExporter; @@ -207,6 +211,8 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY1; + private static Key KEY2; + // Random int generator for trace ID and span ID private static Random random; @@ -233,9 +239,14 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Datastore datastore; + private static RemoteDatastoreHelper remoteDatastoreHelper; + @TestParameter boolean useGlobalOpenTelemetrySDK; - @TestParameter({"default", "test-db"}) + @TestParameter({ + /*(default)*/ + "", "test-db" + }) String datastoreNamedDatabase; @BeforeClass @@ -280,8 +291,7 @@ public void before() throws Exception { // but because gRPC traces need to be deterministically force-flushed for every test String namedDb = datastoreNamedDatabase(); logger.log(Level.INFO, "Integration test using named database " + namedDb); - RemoteDatastoreHelper remoteDatastoreHelper = - RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); + remoteDatastoreHelper = RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); options = remoteDatastoreHelper.getOptions(); datastore = options.getService(); @@ -292,7 +302,14 @@ public void before() throws Exception { String projectId = options.getProjectId(); String kind1 = "kind1"; - KEY1 = Key.newBuilder(projectId, kind1, "name", options.getDatabaseId()).build(); + KEY1 = + Key.newBuilder(projectId, kind1, "key1", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY2 = + Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); // Set up the tracer for custom TraceID injection rootSpanName = @@ -319,6 +336,7 @@ public void after() throws Exception { if (isUsingGlobalOpenTelemetrySDK()) { GlobalOpenTelemetry.resetForTest(); } + remoteDatastoreHelper.deleteNamespace(); rootSpanName = null; tracer = null; retrievedTrace = null; @@ -527,4 +545,32 @@ public void lookupTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_LOOKUP); } + + @Test + public void runQueryTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUNQUERY); + } } From 3fef66fb65d369d5a82d0e47140fb39c4da645cc Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 16 May 2024 10:49:34 -0700 Subject: [PATCH 09/41] Refactor: s/RUNQUERY/RUN_QUERY --- .../google/cloud/datastore/DatastoreImpl.java | 37 +++++++++++++------ .../cloud/datastore/telemetry/TraceUtil.java | 2 - .../cloud/datastore/it/ITE2ETracingTest.java | 4 +- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 401e3ed54..cf84d62b6 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -35,6 +35,7 @@ import com.google.datastore.v1.ExplainOptions; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.ReserveIdsRequest; +import com.google.datastore.v1.RunQueryResponse; import com.google.datastore.v1.TransactionOptions; import com.google.protobuf.ByteString; import io.opencensus.common.Scope; @@ -240,20 +241,34 @@ public AggregationResults runAggregation( com.google.datastore.v1.RunQueryResponse runQuery( final com.google.datastore.v1.RunQueryRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_RUNQUERY); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { - return RetryHelper.runWithRetries( - () -> datastoreRpc.runQuery(requestPb), - retrySettings, - requestPb.getReadOptions().getTransaction().isEmpty() - ? EXCEPTION_HANDLER - : TRANSACTION_OPERATION_EXCEPTION_HANDLER, - getOptions().getClock()); + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUNQUERY); + ReadOptions readOptions = requestPb.getReadOptions(); + span.setAttribute( + "isTransactional", readOptions.hasTransaction() || readOptions.hasNewTransaction()); + span.setAttribute("readConsistency", readOptions.getReadConsistency().toString()); + + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + RunQueryResponse response = + RetryHelper.runWithRetries( + () -> datastoreRpc.runQuery(requestPb), + retrySettings, + requestPb.getReadOptions().getTransaction().isEmpty() + ? EXCEPTION_HANDLER + : TRANSACTION_OPERATION_EXCEPTION_HANDLER, + getOptions().getClock()); + span.addEvent( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUNQUERY + ": Completed", + new ImmutableMap.Builder() + .put("Received", response.getBatch().getEntityResultsCount()) + .put("More results", response.getBatch().getMoreResults().toString()) + .build()); + return response; } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 14d8801fc..e59584958 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -31,9 +31,7 @@ public interface TraceUtil { static final String ATTRIBUTE_SERVICE_PREFIX = "gcp.datastore."; static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; - static final String SPAN_NAME_LOOKUP = "Lookup"; - static final String SPAN_NAME_RUNQUERY = "RunQuery"; /** diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index d498d38ba..8ce565b1e 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -17,7 +17,7 @@ package com.google.cloud.datastore.it; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUNQUERY; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -571,6 +571,6 @@ public void runQueryTraceTest() throws Exception { } waitForTracesToComplete(); - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUNQUERY); + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); } } From fb8a28e7af08baad0468c43eaff25f36959258e4 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 16 May 2024 15:14:25 -0700 Subject: [PATCH 10/41] feat: RunAggregationQuery Trace Instrumentation --- .../cloud/datastore/telemetry/TraceUtil.java | 2 +- .../cloud/datastore/it/ITE2ETracingTest.java | 79 ++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index e59584958..5bf505638 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -32,7 +32,7 @@ public interface TraceUtil { static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; static final String SPAN_NAME_LOOKUP = "Lookup"; - static final String SPAN_NAME_RUNQUERY = "RunQuery"; + static final String SPAN_NAME_RUN_QUERY = "RunQuery"; /** * Creates and returns an instance of the TraceUtil class. diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 8ce565b1e..4b2391725 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -16,8 +16,11 @@ package com.google.cloud.datastore.it; +import static com.google.cloud.datastore.aggregation.Aggregation.count; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; +import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -26,12 +29,16 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.datastore.AggregationQuery; +import com.google.cloud.datastore.AggregationResult; +import com.google.cloud.datastore.AggregationResults; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; @@ -94,6 +101,7 @@ // 5. Traces are read-back using TraceServiceClient and verified against expected Call Stacks. @RunWith(TestParameterInjector.class) public class ITE2ETracingTest { + protected boolean isUsingGlobalOpenTelemetrySDK() { return useGlobalOpenTelemetrySDK; } @@ -213,6 +221,10 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY2; + private static Key KEY3; + + private static Key KEY4; + // Random int generator for trace ID and span ID private static Random random; @@ -307,10 +319,17 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY2 = + Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY3 = + Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY4 = Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); - // Set up the tracer for custom TraceID injection rootSpanName = String.format("%s%d", this.getClass().getSimpleName(), System.currentTimeMillis()); @@ -573,4 +592,62 @@ public void runQueryTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); } + + @Test + public void runAggregationQueryTraceTest() throws Exception { + Entity entity1 = + Entity.newBuilder(KEY1) + .set("pepper_name", "jalapeno") + .set("max_scoville_level", 10000) + .build(); + Entity entity2 = + Entity.newBuilder(KEY2) + .set("pepper_name", "serrano") + .set("max_scoville_level", 25000) + .build(); + Entity entity3 = + Entity.newBuilder(KEY3) + .set("pepper_name", "habanero") + .set("max_scoville_level", 350000) + .build(); + Entity entity4 = + Entity.newBuilder(KEY4) + .set("pepper_name", "ghost") + .set("max_scoville_level", 1500000) + .build(); + + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + entityList.add(entity3); + entityList.add(entity4); + + List response = datastore.add(entity1, entity2, entity3, entity4); + assertEquals(entityList, response); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter mediumSpicyFilters = PropertyFilter.lt("max_scoville_level", 100000); + StructuredQuery mediumSpicyQuery = + Query.newEntityQueryBuilder() + .setKind(KEY1.getKind()) + .setFilter(mediumSpicyFilters) + .build(); + AggregationQuery countSpicyPeppers = + Query.newAggregationQueryBuilder() + .addAggregation(count().as("count")) + .over(mediumSpicyQuery) + .build(); + AggregationResults results = datastore.runAggregation(countSpicyPeppers); + assertThat(results.size()).isEqualTo(1); + AggregationResult result = results.get(0); + assertThat(result.getLong("count")).isEqualTo(2L); + } finally { + rootSpan.end(); + } + + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_AGGREGATION_QUERY); + } } From 02d178d98380cdaa75aa39a741b39177511006f4 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 16 May 2024 16:13:14 -0700 Subject: [PATCH 11/41] Build: retiring test assertions for OpenCensus spans - will be replacing this in hermetic integration tests for OpenTelemetry using in-memory span exports (in addition to ITE2ETraceTest.java). --- .../datastore/RetryAndTraceDatastoreRpcDecoratorTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java index b01bcab82..13b63a037 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java @@ -26,6 +26,10 @@ import com.google.cloud.datastore.spi.v1.DatastoreRpc; import com.google.datastore.v1.RunAggregationQueryRequest; import com.google.datastore.v1.RunAggregationQueryResponse; +<<<<<<< HEAD +======= + +>>>>>>> ec777fc (Build: retiring test assertions for OpenCensus spans - will be replacing this in hermetic integration tests for OpenTelemetry using in-memory span exports (in addition to ITE2ETraceTest.java).) import org.junit.Before; import org.junit.Test; From a2a2f03313c98cf5e669aefeb7e6c76833f32570 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Fri, 17 May 2024 09:48:52 -0700 Subject: [PATCH 12/41] Formatting --- ...etryAndTraceDatastoreRpcDecoratorTest.java | 78 ------------------- 1 file changed, 78 deletions(-) delete mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java deleted file mode 100644 index 13b63a037..000000000 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecoratorTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.datastore; - -import static com.google.common.truth.Truth.assertThat; -import static com.google.rpc.Code.UNAVAILABLE; -import static org.easymock.EasyMock.createStrictMock; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.verify; - -import com.google.api.gax.retrying.RetrySettings; -import com.google.cloud.datastore.spi.v1.DatastoreRpc; -import com.google.datastore.v1.RunAggregationQueryRequest; -import com.google.datastore.v1.RunAggregationQueryResponse; -<<<<<<< HEAD -======= - ->>>>>>> ec777fc (Build: retiring test assertions for OpenCensus spans - will be replacing this in hermetic integration tests for OpenTelemetry using in-memory span exports (in addition to ITE2ETraceTest.java).) -import org.junit.Before; -import org.junit.Test; - -public class RetryAndTraceDatastoreRpcDecoratorTest { - - public static final int MAX_ATTEMPTS = 3; - private DatastoreRpc mockDatastoreRpc; - private TraceUtil mockTraceUtil; - private DatastoreOptions datastoreOptions = - DatastoreOptions.newBuilder().setProjectId("project-id").build(); - private RetrySettings retrySettings = - RetrySettings.newBuilder().setMaxAttempts(MAX_ATTEMPTS).build(); - - private RetryAndTraceDatastoreRpcDecorator datastoreRpcDecorator; - - @Before - public void setUp() throws Exception { - mockDatastoreRpc = createStrictMock(DatastoreRpc.class); - datastoreRpcDecorator = - new RetryAndTraceDatastoreRpcDecorator( - mockDatastoreRpc, mockTraceUtil, retrySettings, datastoreOptions); - } - - @Test - public void testRunAggregationQuery() { - RunAggregationQueryRequest aggregationQueryRequest = - RunAggregationQueryRequest.getDefaultInstance(); - RunAggregationQueryResponse aggregationQueryResponse = - RunAggregationQueryResponse.getDefaultInstance(); - - expect(mockDatastoreRpc.runAggregationQuery(aggregationQueryRequest)) - .andThrow( - new DatastoreException( - UNAVAILABLE.getNumber(), "API not accessible currently", UNAVAILABLE.name())) - .times(2) - .andReturn(aggregationQueryResponse); - - replay(mockDatastoreRpc); - - RunAggregationQueryResponse actualAggregationQueryResponse = - datastoreRpcDecorator.runAggregationQuery(aggregationQueryRequest); - - assertThat(actualAggregationQueryResponse).isSameInstanceAs(aggregationQueryResponse); - verify(mockDatastoreRpc); - } -} From 73241a26a01db7ea75a9adf18f50bf2f4c7e4859 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 20 May 2024 11:37:22 -0700 Subject: [PATCH 13/41] Fixing @Test annotation missed after merge --- .../cloud/datastore/it/ITE2ETracingTest.java | 151 +++++++++--------- 1 file changed, 79 insertions(+), 72 deletions(-) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 4b2391725..f33a7e23a 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -16,11 +16,9 @@ package com.google.cloud.datastore.it; -import static com.google.cloud.datastore.aggregation.Aggregation.count; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; -import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -29,16 +27,12 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.rpc.NotFoundException; -import com.google.cloud.datastore.AggregationQuery; -import com.google.cloud.datastore.AggregationResult; -import com.google.cloud.datastore.AggregationResults; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; @@ -101,7 +95,6 @@ // 5. Traces are read-back using TraceServiceClient and verified against expected Call Stacks. @RunWith(TestParameterInjector.class) public class ITE2ETracingTest { - protected boolean isUsingGlobalOpenTelemetrySDK() { return useGlobalOpenTelemetrySDK; } @@ -221,10 +214,6 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY2; - private static Key KEY3; - - private static Key KEY4; - // Random int generator for trace ID and span ID private static Random random; @@ -256,8 +245,9 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack @TestParameter boolean useGlobalOpenTelemetrySDK; @TestParameter({ - /*(default)*/ - "", "test-db" + /*(default)*/ + "", + "test-db" }) String datastoreNamedDatabase; @@ -319,17 +309,10 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY2 = - Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) - .setNamespace(options.getNamespace()) - .build(); - KEY3 = - Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) - .setNamespace(options.getNamespace()) - .build(); - KEY4 = Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); + // Set up the tracer for custom TraceID injection rootSpanName = String.format("%s%d", this.getClass().getSimpleName(), System.currentTimeMillis()); @@ -566,7 +549,45 @@ public void lookupTraceTest() throws Exception { } @Test - public void runQueryTraceTest() throws Exception { + public void commitTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + try (Scope ignored = rootSpan.makeCurrent()) { + Entity response = datastore.add(entity1); + assertEquals(entity1, response); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } + + @Test + public void putTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + try (Scope ignored = rootSpan.makeCurrent()) { + Entity response = datastore.put(entity1); + assertEquals(entity1, response); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } + + @Test + public void updateTraceTest() throws Exception { + assertNotNull(customSpanContext); + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); List entityList = new ArrayList<>(); @@ -578,76 +599,62 @@ public void runQueryTraceTest() throws Exception { Span rootSpan = getNewRootSpanWithContext(); try (Scope ignored = rootSpan.makeCurrent()) { - PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); - Query query = - Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); - QueryResults queryResults = datastore.run(query); - assertTrue(queryResults.hasNext()); - assertEquals(entity1, queryResults.next()); - assertFalse(queryResults.hasNext()); + Entity entity1_update = + Entity.newBuilder(entity1).set("test_field", "new_test_value1").build(); + Entity entity2_update = + Entity.newBuilder(entity2).set("test_field", "new_test_value1").build(); + datastore.update(entity1_update, entity2_update); } finally { rootSpan.end(); } waitForTracesToComplete(); - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); } @Test - public void runAggregationQueryTraceTest() throws Exception { - Entity entity1 = - Entity.newBuilder(KEY1) - .set("pepper_name", "jalapeno") - .set("max_scoville_level", 10000) - .build(); - Entity entity2 = - Entity.newBuilder(KEY2) - .set("pepper_name", "serrano") - .set("max_scoville_level", 25000) - .build(); - Entity entity3 = - Entity.newBuilder(KEY3) - .set("pepper_name", "habanero") - .set("max_scoville_level", 350000) - .build(); - Entity entity4 = - Entity.newBuilder(KEY4) - .set("pepper_name", "ghost") - .set("max_scoville_level", 1500000) - .build(); + public void deleteTraceTest() throws Exception { + assertNotNull(customSpanContext); + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + Entity response = datastore.put(entity1); + assertEquals(entity1, response); + + Span rootSpan = getNewRootSpanWithContext(); + + try (Scope ignored = rootSpan.makeCurrent()) { + datastore.delete(entity1.getKey()); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); + } + + public void runQueryTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); List entityList = new ArrayList<>(); entityList.add(entity1); entityList.add(entity2); - entityList.add(entity3); - entityList.add(entity4); - List response = datastore.add(entity1, entity2, entity3, entity4); + List response = datastore.add(entity1, entity2); assertEquals(entityList, response); Span rootSpan = getNewRootSpanWithContext(); try (Scope ignored = rootSpan.makeCurrent()) { - PropertyFilter mediumSpicyFilters = PropertyFilter.lt("max_scoville_level", 100000); - StructuredQuery mediumSpicyQuery = - Query.newEntityQueryBuilder() - .setKind(KEY1.getKind()) - .setFilter(mediumSpicyFilters) - .build(); - AggregationQuery countSpicyPeppers = - Query.newAggregationQueryBuilder() - .addAggregation(count().as("count")) - .over(mediumSpicyQuery) - .build(); - AggregationResults results = datastore.runAggregation(countSpicyPeppers); - assertThat(results.size()).isEqualTo(1); - AggregationResult result = results.get(0); - assertThat(result.getLong("count")).isEqualTo(2L); + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); } finally { rootSpan.end(); } - waitForTracesToComplete(); - fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_AGGREGATION_QUERY); + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); } } From 75a98b4a4b9dfd543b4d2861d3a3b40fdb1de9e1 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 20 May 2024 14:01:37 -0700 Subject: [PATCH 14/41] Formatting --- .../java/com/google/cloud/datastore/it/ITE2ETracingTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index f33a7e23a..9a4a99106 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -632,7 +632,7 @@ public void deleteTraceTest() throws Exception { } public void runQueryTraceTest() throws Exception { - Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); List entityList = new ArrayList<>(); entityList.add(entity1); From 585181fddbdfe851f01a58d939d2ea4190cefa97 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 20 May 2024 17:17:11 -0700 Subject: [PATCH 15/41] feat: Add Transaction tracing test: transactionalLookupTest --- .../google/cloud/datastore/DatastoreImpl.java | 15 ++-- .../RetryAndTraceDatastoreRpcDecorator.java | 12 ++- .../cloud/datastore/telemetry/TraceUtil.java | 2 + .../cloud/datastore/it/ITE2ETracingTest.java | 80 ++++++++++++++++++- 4 files changed, 99 insertions(+), 10 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index cf84d62b6..ebf5aa7e9 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -242,7 +242,7 @@ public AggregationResults runAggregation( com.google.datastore.v1.RunQueryResponse runQuery( final com.google.datastore.v1.RunQueryRequest requestPb) { com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUNQUERY); + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY); ReadOptions readOptions = requestPb.getReadOptions(); span.setAttribute( "isTransactional", readOptions.hasTransaction() || readOptions.hasNewTransaction()); @@ -258,7 +258,7 @@ com.google.datastore.v1.RunQueryResponse runQuery( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); span.addEvent( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUNQUERY + ": Completed", + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY + ": Completed", new ImmutableMap.Builder() .put("Received", response.getBatch().getEntityResultsCount()) .put("More results", response.getBatch().getMoreResults().toString()) @@ -634,8 +634,11 @@ private com.google.datastore.v1.CommitResponse commitMutation( com.google.datastore.v1.CommitResponse commit( final com.google.datastore.v1.CommitRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_COMMIT); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT); + span.setAttribute("isTransactional", requestPb.hasTransaction()); + + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.commit(requestPb), retrySettings, @@ -644,10 +647,10 @@ com.google.datastore.v1.CommitResponse commit( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java index ba4adb5ff..7abf1d360 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java @@ -30,6 +30,7 @@ import com.google.datastore.v1.CommitResponse; import com.google.datastore.v1.LookupRequest; import com.google.datastore.v1.LookupResponse; +import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.ReserveIdsRequest; import com.google.datastore.v1.ReserveIdsResponse; import com.google.datastore.v1.RollbackRequest; @@ -101,9 +102,14 @@ public RunQueryResponse runQuery(RunQueryRequest request) { @Override public RunAggregationQueryResponse runAggregationQuery(RunAggregationQueryRequest request) { - return invokeRpc( - () -> datastoreRpc.runAggregationQuery(request), - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY); + ReadOptions readOptions = request.getReadOptions(); + boolean isTransactional = readOptions.hasTransaction() || readOptions.hasNewTransaction(); + String spanName = + (isTransactional + ? com.google.cloud.datastore.telemetry.TraceUtil + .SPAN_NAME_TRANSACTION_RUN_AGGREGATION_QUERY + : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY); + return invokeRpc(() -> datastoreRpc.runAggregationQuery(request), spanName); } public O invokeRpc(Callable block, String startSpan) { diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 5bf505638..00e9f2376 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -32,7 +32,9 @@ public interface TraceUtil { static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; static final String SPAN_NAME_LOOKUP = "Lookup"; + static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; + static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; /** * Creates and returns an instance of the TraceUtil class. diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 9a4a99106..357e748f1 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -16,9 +16,12 @@ package com.google.cloud.datastore.it; +import static com.google.cloud.datastore.aggregation.Aggregation.count; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; +import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -27,12 +30,16 @@ import static org.junit.Assert.assertTrue; import com.google.api.gax.rpc.NotFoundException; +import com.google.cloud.datastore.AggregationQuery; +import com.google.cloud.datastore.AggregationResult; +import com.google.cloud.datastore.AggregationResults; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; @@ -95,6 +102,7 @@ // 5. Traces are read-back using TraceServiceClient and verified against expected Call Stacks. @RunWith(TestParameterInjector.class) public class ITE2ETracingTest { + protected boolean isUsingGlobalOpenTelemetrySDK() { return useGlobalOpenTelemetrySDK; } @@ -214,6 +222,10 @@ private boolean dfsContainsCallStack(long spanId, List expectedCallStack private static Key KEY2; + private static Key KEY3; + + private static Key KEY4; + // Random int generator for trace ID and span ID private static Random random; @@ -309,10 +321,17 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY2 = + Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY3 = + Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY4 = Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); - // Set up the tracer for custom TraceID injection rootSpanName = String.format("%s%d", this.getClass().getSimpleName(), System.currentTimeMillis()); @@ -631,6 +650,7 @@ public void deleteTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_COMMIT); } + @Test public void runQueryTraceTest() throws Exception { Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); @@ -657,4 +677,62 @@ public void runQueryTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_QUERY); } + + @Test + public void runAggregationQueryTraceTest() throws Exception { + Entity entity1 = + Entity.newBuilder(KEY1) + .set("pepper_name", "jalapeno") + .set("max_scoville_level", 10000) + .build(); + Entity entity2 = + Entity.newBuilder(KEY2) + .set("pepper_name", "serrano") + .set("max_scoville_level", 25000) + .build(); + Entity entity3 = + Entity.newBuilder(KEY3) + .set("pepper_name", "habanero") + .set("max_scoville_level", 350000) + .build(); + Entity entity4 = + Entity.newBuilder(KEY4) + .set("pepper_name", "ghost") + .set("max_scoville_level", 1500000) + .build(); + + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + entityList.add(entity3); + entityList.add(entity4); + + List response = datastore.add(entity1, entity2, entity3, entity4); + assertEquals(entityList, response); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter mediumSpicyFilters = PropertyFilter.lt("max_scoville_level", 100000); + StructuredQuery mediumSpicyQuery = + Query.newEntityQueryBuilder() + .setKind(KEY1.getKind()) + .setFilter(mediumSpicyFilters) + .build(); + AggregationQuery countSpicyPeppers = + Query.newAggregationQueryBuilder() + .addAggregation(count().as("count")) + .over(mediumSpicyQuery) + .build(); + AggregationResults results = datastore.runAggregation(countSpicyPeppers); + assertThat(results.size()).isEqualTo(1); + AggregationResult result = results.get(0); + assertThat(result.getLong("count")).isEqualTo(2L); + } finally { + rootSpan.end(); + } + + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_AGGREGATION_QUERY); + } } From 4edc2ef7e02ab0be1db794f76f4793bad1bcb553 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Tue, 21 May 2024 10:10:17 -0700 Subject: [PATCH 16/41] test: Transaction test for RunInTransaction - need to fix trace instrumentation for RunIn.. --- .../cloud/datastore/it/ITE2ETracingTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 357e748f1..ecb2a84da 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -17,10 +17,12 @@ package com.google.cloud.datastore.it; import static com.google.cloud.datastore.aggregation.Aggregation.count; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP; import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; @@ -39,8 +41,10 @@ import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.ReadOption; import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; +import com.google.cloud.datastore.Transaction; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.cloud.opentelemetry.trace.TraceConfiguration; import com.google.cloud.opentelemetry.trace.TraceExporter; @@ -67,6 +71,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Random; @@ -735,4 +740,40 @@ public void runAggregationQueryTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RUN_AGGREGATION_QUERY); } + + @Test + public void transactionalLookupTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + Transaction transaction = datastore.newTransaction(); + Entity entity = datastore.get(KEY1, ReadOption.transactionId(transaction.getTransactionId())); + assertNull(entity); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace( + customSpanContext.getTraceId(), + /*numExpectedSpans=*/ 2, + Collections.singletonList( + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION))); + + fetchAndValidateTrace( + customSpanContext.getTraceId(), + /*numExpectedSpans=*/ 2, + Collections.singletonList( + Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP))); + } + + @Test + public void runInTransactionQueryTest() throws Exception {} + + @Test + public void runInTransactionAggregationQueryTest() throws Exception {} + + @Test + public void readWriteTransactionTraceTest() throws Exception {} } From 22571ea506a77c5ed0fe7942bc66608055346719 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 23 May 2024 11:28:48 -0700 Subject: [PATCH 17/41] Adding transaction span names --- .../java/com/google/cloud/datastore/telemetry/TraceUtil.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 00e9f2376..4cca0d7ae 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -35,7 +35,10 @@ public interface TraceUtil { static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; - + static final String SPAN_NAME_BEGIN_TRANSACTION = "Transaction.Begin"; + static final String SPAN_NAME_TRANSACTION_LOOKUP = "Transaction.Lookup"; + static final String SPAN_NAME_TRANSACTION_COMMIT = "Transaction.Commit"; + static final String SPAN_NAME_ROLLBACK = "Transaction.Rollback"; /** * Creates and returns an instance of the TraceUtil class. * From 3076735c4f818043ffe3b5812cfe35cc98c8318e Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 23 May 2024 12:43:32 -0700 Subject: [PATCH 18/41] TransactionLookupTest --- .../google/cloud/datastore/DatastoreImpl.java | 54 +++++++++---------- .../cloud/datastore/telemetry/TraceUtil.java | 2 + .../cloud/datastore/it/ITE2ETracingTest.java | 6 +-- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index ebf5aa7e9..66bc6c0ba 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -469,27 +469,27 @@ protected Entity computeNext() { com.google.datastore.v1.LookupResponse lookup( final com.google.datastore.v1.LookupRequest requestPb) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP); - final ReadOptions readOptions = requestPb.getReadOptions(); - span.setAttribute( - "isTransactional", (readOptions.hasTransaction() || readOptions.hasNewTransaction())); + ReadOptions readOptions = requestPb.getReadOptions(); + boolean isTransactional = readOptions.hasTransaction() || readOptions.hasNewTransaction(); + String spanName = + (isTransactional + ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP + : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP); + com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); + span.setAttribute("isTransactional", isTransactional); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( - new Callable() { - @Override - public com.google.datastore.v1.LookupResponse call() throws DatastoreException { - com.google.datastore.v1.LookupResponse response = datastoreRpc.lookup(requestPb); - span.addEvent( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP + ": Completed", - new ImmutableMap.Builder() - .put("Received", response.getFoundCount()) - .put("Missing", response.getMissingCount()) - .put("Deferred", response.getDeferredCount()) - .build()); - return response; - } + () -> { + com.google.datastore.v1.LookupResponse response = datastoreRpc.lookup(requestPb); + span.addEvent( + spanName + ": Completed", + new ImmutableMap.Builder() + .put("Received", response.getFoundCount()) + .put("Missing", response.getMissingCount()) + .put("Deferred", response.getDeferredCount()) + .build()); + return response; }, retrySettings, requestPb.getReadOptions().getTransaction().isEmpty() @@ -661,24 +661,20 @@ ByteString requestTransactionId( com.google.datastore.v1.BeginTransactionResponse beginTransaction( final com.google.datastore.v1.BeginTransactionRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_BEGINTRANSACTION); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope scope = span.makeCurrent()) { return RetryHelper.runWithRetries( - new Callable() { - @Override - public com.google.datastore.v1.BeginTransactionResponse call() - throws DatastoreException { - return datastoreRpc.beginTransaction(requestPb); - } - }, + () -> datastoreRpc.beginTransaction(requestPb), retrySettings, EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 4cca0d7ae..d43c36a07 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -39,6 +39,8 @@ public interface TraceUtil { static final String SPAN_NAME_TRANSACTION_LOOKUP = "Transaction.Lookup"; static final String SPAN_NAME_TRANSACTION_COMMIT = "Transaction.Commit"; static final String SPAN_NAME_ROLLBACK = "Transaction.Rollback"; + static final String SPAN_NAME_TRANSACTION_RUN_AGGREGATION_QUERY = + "Transaction.RunAggregationQuery"; /** * Creates and returns an instance of the TraceUtil class. * diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index ecb2a84da..67df8c44b 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -758,14 +758,12 @@ public void transactionalLookupTest() throws Exception { fetchAndValidateTrace( customSpanContext.getTraceId(), /*numExpectedSpans=*/ 2, - Collections.singletonList( - Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION))); + Collections.singletonList(Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION))); fetchAndValidateTrace( customSpanContext.getTraceId(), /*numExpectedSpans=*/ 2, - Collections.singletonList( - Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP))); + Collections.singletonList(Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP))); } @Test From 8dca8671a79043eb6b322e2d9c509be90071c607 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 29 May 2024 17:24:03 -0700 Subject: [PATCH 19/41] feat: support for transactional operations - tested using newTransaction() and runInTransaction() --- .../google/cloud/datastore/DatastoreImpl.java | 49 +++++++++++------ .../cloud/datastore/TransactionImpl.java | 10 +++- .../cloud/datastore/telemetry/TraceUtil.java | 2 + .../cloud/datastore/it/ITE2ETracingTest.java | 54 ++++++++++++++++--- 4 files changed, 91 insertions(+), 24 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 66bc6c0ba..753dab596 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -25,6 +25,7 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.datastore.execution.AggregationQueryExecutor; import com.google.cloud.datastore.spi.v1.DatastoreRpc; +import com.google.cloud.datastore.telemetry.TraceUtil.Context; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.AbstractIterator; @@ -52,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import javax.annotation.Nonnull; final class DatastoreImpl extends BaseService implements Datastore { @@ -103,13 +105,18 @@ static class ReadWriteTransactionCallable implements Callable { private final TransactionCallable callable; private volatile TransactionOptions options; private volatile Transaction transaction; + @Nonnull private final Context transactionTraceContext; ReadWriteTransactionCallable( - Datastore datastore, TransactionCallable callable, TransactionOptions options) { + Datastore datastore, + TransactionCallable callable, + TransactionOptions options, + @Nonnull Context parentTraceContext) { this.datastore = datastore; this.callable = callable; this.options = options; this.transaction = null; + this.transactionTraceContext = parentTraceContext; } Datastore getDatastore() { @@ -132,8 +139,9 @@ void setPrevTransactionId(ByteString transactionId) { @Override public T call() throws DatastoreException { - transaction = datastore.newTransaction(options); - try { + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = + transactionTraceContext.makeCurrent()) { + transaction = datastore.newTransaction(options); T value = callable.run(transaction); transaction.commit(); return value; @@ -154,36 +162,41 @@ public T call() throws DatastoreException { @Override public T runInTransaction(final TransactionCallable callable) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_TRANSACTION); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( - new ReadWriteTransactionCallable(this, callable, null), + new ReadWriteTransactionCallable(this, callable, null, otelTraceUtil.currentContext()), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } @Override public T runInTransaction( final TransactionCallable callable, TransactionOptions transactionOptions) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_TRANSACTION); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( - new ReadWriteTransactionCallable(this, callable, transactionOptions), + new ReadWriteTransactionCallable( + this, callable, transactionOptions, otelTraceUtil.currentContext()), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } @@ -634,10 +647,14 @@ private com.google.datastore.v1.CommitResponse commitMutation( com.google.datastore.v1.CommitResponse commit( final com.google.datastore.v1.CommitRequest requestPb) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT); + final boolean isTransactional = + requestPb.hasTransaction() || requestPb.hasSingleUseTransaction(); + final String spanName = + isTransactional + ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT + : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; + com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); span.setAttribute("isTransactional", requestPb.hasTransaction()); - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.commit(requestPb), diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java index f08a908ec..b87d175a0 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java @@ -20,6 +20,8 @@ import com.google.api.core.BetaApi; import com.google.cloud.datastore.models.ExplainOptions; +import com.google.cloud.datastore.telemetry.TraceUtil.Context; +import com.google.cloud.datastore.telemetry.TraceUtil.Scope; import com.google.common.collect.ImmutableList; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.TransactionOptions; @@ -28,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import javax.annotation.Nonnull; final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction { @@ -37,6 +40,8 @@ final class TransactionImpl extends BaseDatastoreBatchWriter implements Transact private final ReadOptionProtoPreparer readOptionProtoPreparer; + @Nonnull private final Context transactionTraceContext; + static class ResponseImpl implements Transaction.Response { private final com.google.datastore.v1.CommitResponse response; @@ -78,6 +83,7 @@ public List getGeneratedKeys() { transactionId = datastore.requestTransactionId(requestPb); this.readOptionProtoPreparer = new ReadOptionProtoPreparer(); + this.transactionTraceContext = datastore.getOptions().getTraceUtil().currentContext(); } @Override @@ -96,7 +102,9 @@ public Iterator get(Key... keys) { @Override public List fetch(Key... keys) { validateActive(); - return DatastoreHelper.fetch(this, keys); + try (Scope ignored = transactionTraceContext.makeCurrent()) { + return DatastoreHelper.fetch(this, keys); + } } @Override diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index d43c36a07..fe0e92161 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -21,6 +21,7 @@ import com.google.api.core.InternalExtensionOnly; import com.google.cloud.datastore.DatastoreOptions; import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.context.Context; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -35,6 +36,7 @@ public interface TraceUtil { static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; + static final String SPAN_NAME_TRANSACTION_RUN = "Transaction.run"; static final String SPAN_NAME_BEGIN_TRANSACTION = "Transaction.Begin"; static final String SPAN_NAME_TRANSACTION_LOOKUP = "Transaction.Lookup"; static final String SPAN_NAME_TRANSACTION_COMMIT = "Transaction.Commit"; diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 67df8c44b..d2cffc5db 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -22,7 +22,9 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN; import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; @@ -749,6 +751,7 @@ public void transactionalLookupTest() throws Exception { try (Scope ignored = rootSpan.makeCurrent()) { Transaction transaction = datastore.newTransaction(); Entity entity = datastore.get(KEY1, ReadOption.transactionId(transaction.getTransactionId())); + transaction.commit(); assertNull(entity); } finally { rootSpan.end(); @@ -757,18 +760,55 @@ public void transactionalLookupTest() throws Exception { fetchAndValidateTrace( customSpanContext.getTraceId(), - /*numExpectedSpans=*/ 2, - Collections.singletonList(Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION))); + /*numExpectedSpans=*/ 3, + Arrays.asList( + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP), + Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); + } + + @Test + public void runInTransactionQueryTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + Datastore.TransactionCallable callable = + transaction -> { + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + return true; + }; + datastore.runInTransaction(callable); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); fetchAndValidateTrace( customSpanContext.getTraceId(), - /*numExpectedSpans=*/ 2, - Collections.singletonList(Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP))); + /*numExpectedSpans=*/ 4, + Arrays.asList( + Collections.singletonList(SPAN_NAME_TRANSACTION_RUN), + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_RUN_QUERY), + Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); } - @Test - public void runInTransactionQueryTest() throws Exception {} - @Test public void runInTransactionAggregationQueryTest() throws Exception {} From 66cebbb4daae26a9a6b1d14cae0272611af68063 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 29 May 2024 17:31:37 -0700 Subject: [PATCH 20/41] Revert "feat: support for transactional operations" This reverts commit 10341c0b97cbc2025f9f928ce8cb09d5c036a5b3. --- .../google/cloud/datastore/DatastoreImpl.java | 49 ++++++----------- .../cloud/datastore/TransactionImpl.java | 10 +--- .../cloud/datastore/telemetry/TraceUtil.java | 2 - .../cloud/datastore/it/ITE2ETracingTest.java | 54 +++---------------- 4 files changed, 24 insertions(+), 91 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 753dab596..66bc6c0ba 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -25,7 +25,6 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.datastore.execution.AggregationQueryExecutor; import com.google.cloud.datastore.spi.v1.DatastoreRpc; -import com.google.cloud.datastore.telemetry.TraceUtil.Context; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.AbstractIterator; @@ -53,7 +52,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; -import javax.annotation.Nonnull; final class DatastoreImpl extends BaseService implements Datastore { @@ -105,18 +103,13 @@ static class ReadWriteTransactionCallable implements Callable { private final TransactionCallable callable; private volatile TransactionOptions options; private volatile Transaction transaction; - @Nonnull private final Context transactionTraceContext; ReadWriteTransactionCallable( - Datastore datastore, - TransactionCallable callable, - TransactionOptions options, - @Nonnull Context parentTraceContext) { + Datastore datastore, TransactionCallable callable, TransactionOptions options) { this.datastore = datastore; this.callable = callable; this.options = options; this.transaction = null; - this.transactionTraceContext = parentTraceContext; } Datastore getDatastore() { @@ -139,9 +132,8 @@ void setPrevTransactionId(ByteString transactionId) { @Override public T call() throws DatastoreException { - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = - transactionTraceContext.makeCurrent()) { - transaction = datastore.newTransaction(options); + transaction = datastore.newTransaction(options); + try { T value = callable.run(transaction); transaction.commit(); return value; @@ -162,41 +154,36 @@ public T call() throws DatastoreException { @Override public T runInTransaction(final TransactionCallable callable) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_TRANSACTION); + try (Scope scope = traceUtil.getTracer().withSpan(span)) { return RetryHelper.runWithRetries( - new ReadWriteTransactionCallable(this, callable, null, otelTraceUtil.currentContext()), + new ReadWriteTransactionCallable(this, callable, null), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.end(e); + span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); throw DatastoreException.translateAndThrow(e); } finally { - span.end(); + span.end(TraceUtil.END_SPAN_OPTIONS); } } @Override public T runInTransaction( final TransactionCallable callable, TransactionOptions transactionOptions) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_TRANSACTION); + try (Scope scope = traceUtil.getTracer().withSpan(span)) { return RetryHelper.runWithRetries( - new ReadWriteTransactionCallable( - this, callable, transactionOptions, otelTraceUtil.currentContext()), + new ReadWriteTransactionCallable(this, callable, transactionOptions), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.end(e); + span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); throw DatastoreException.translateAndThrow(e); } finally { - span.end(); + span.end(TraceUtil.END_SPAN_OPTIONS); } } @@ -647,14 +634,10 @@ private com.google.datastore.v1.CommitResponse commitMutation( com.google.datastore.v1.CommitResponse commit( final com.google.datastore.v1.CommitRequest requestPb) { - final boolean isTransactional = - requestPb.hasTransaction() || requestPb.hasSingleUseTransaction(); - final String spanName = - isTransactional - ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT - : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; - com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT); span.setAttribute("isTransactional", requestPb.hasTransaction()); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.commit(requestPb), diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java index b87d175a0..f08a908ec 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java @@ -20,8 +20,6 @@ import com.google.api.core.BetaApi; import com.google.cloud.datastore.models.ExplainOptions; -import com.google.cloud.datastore.telemetry.TraceUtil.Context; -import com.google.cloud.datastore.telemetry.TraceUtil.Scope; import com.google.common.collect.ImmutableList; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.TransactionOptions; @@ -30,7 +28,6 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; -import javax.annotation.Nonnull; final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction { @@ -40,8 +37,6 @@ final class TransactionImpl extends BaseDatastoreBatchWriter implements Transact private final ReadOptionProtoPreparer readOptionProtoPreparer; - @Nonnull private final Context transactionTraceContext; - static class ResponseImpl implements Transaction.Response { private final com.google.datastore.v1.CommitResponse response; @@ -83,7 +78,6 @@ public List getGeneratedKeys() { transactionId = datastore.requestTransactionId(requestPb); this.readOptionProtoPreparer = new ReadOptionProtoPreparer(); - this.transactionTraceContext = datastore.getOptions().getTraceUtil().currentContext(); } @Override @@ -102,9 +96,7 @@ public Iterator get(Key... keys) { @Override public List fetch(Key... keys) { validateActive(); - try (Scope ignored = transactionTraceContext.makeCurrent()) { - return DatastoreHelper.fetch(this, keys); - } + return DatastoreHelper.fetch(this, keys); } @Override diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index fe0e92161..d43c36a07 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -21,7 +21,6 @@ import com.google.api.core.InternalExtensionOnly; import com.google.cloud.datastore.DatastoreOptions; import io.grpc.ManagedChannelBuilder; -import io.opentelemetry.context.Context; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -36,7 +35,6 @@ public interface TraceUtil { static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; - static final String SPAN_NAME_TRANSACTION_RUN = "Transaction.run"; static final String SPAN_NAME_BEGIN_TRANSACTION = "Transaction.Begin"; static final String SPAN_NAME_TRANSACTION_LOOKUP = "Transaction.Lookup"; static final String SPAN_NAME_TRANSACTION_COMMIT = "Transaction.Commit"; diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index d2cffc5db..67df8c44b 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -22,9 +22,7 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP; -import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN; import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; @@ -751,7 +749,6 @@ public void transactionalLookupTest() throws Exception { try (Scope ignored = rootSpan.makeCurrent()) { Transaction transaction = datastore.newTransaction(); Entity entity = datastore.get(KEY1, ReadOption.transactionId(transaction.getTransactionId())); - transaction.commit(); assertNull(entity); } finally { rootSpan.end(); @@ -760,55 +757,18 @@ public void transactionalLookupTest() throws Exception { fetchAndValidateTrace( customSpanContext.getTraceId(), - /*numExpectedSpans=*/ 3, - Arrays.asList( - Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), - Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP), - Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); - } - - @Test - public void runInTransactionQueryTest() throws Exception { - Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); - Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); - List entityList = new ArrayList<>(); - entityList.add(entity1); - entityList.add(entity2); - - List response = datastore.add(entity1, entity2); - assertEquals(entityList, response); - - assertNotNull(customSpanContext); - - Span rootSpan = getNewRootSpanWithContext(); - try (Scope ignored = rootSpan.makeCurrent()) { - PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); - Query query = - Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); - Datastore.TransactionCallable callable = - transaction -> { - QueryResults queryResults = datastore.run(query); - assertTrue(queryResults.hasNext()); - assertEquals(entity1, queryResults.next()); - assertFalse(queryResults.hasNext()); - return true; - }; - datastore.runInTransaction(callable); - } finally { - rootSpan.end(); - } - waitForTracesToComplete(); + /*numExpectedSpans=*/ 2, + Collections.singletonList(Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION))); fetchAndValidateTrace( customSpanContext.getTraceId(), - /*numExpectedSpans=*/ 4, - Arrays.asList( - Collections.singletonList(SPAN_NAME_TRANSACTION_RUN), - Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), - Collections.singletonList(SPAN_NAME_RUN_QUERY), - Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); + /*numExpectedSpans=*/ 2, + Collections.singletonList(Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP))); } + @Test + public void runInTransactionQueryTest() throws Exception {} + @Test public void runInTransactionAggregationQueryTest() throws Exception {} From 24779b9cfbd584ae003f0aa12ce11be3f18ce276 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 6 Jun 2024 21:33:28 -0700 Subject: [PATCH 21/41] feat: support for transactional operations (#1468) * feat: support for transactional operations - tested using newTransaction() and runInTransaction() --- .../google/cloud/datastore/DatastoreImpl.java | 58 +++++++++++++------ .../cloud/datastore/TransactionImpl.java | 5 ++ .../telemetry/DisabledTraceUtil.java | 4 +- .../datastore/telemetry/EnabledTraceUtil.java | 4 +- .../cloud/datastore/telemetry/TraceUtil.java | 5 +- .../cloud/datastore/it/ITE2ETracingTest.java | 54 ++++++++++++++--- .../telemetry/DisabledTraceUtilTest.java | 11 ++-- .../telemetry/EnabledTraceUtilTest.java | 13 +++-- 8 files changed, 113 insertions(+), 41 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 66bc6c0ba..eea94cc60 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -25,6 +25,7 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.datastore.execution.AggregationQueryExecutor; import com.google.cloud.datastore.spi.v1.DatastoreRpc; +import com.google.cloud.datastore.telemetry.TraceUtil.Context; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.AbstractIterator; @@ -52,6 +53,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; +import javax.annotation.Nonnull; final class DatastoreImpl extends BaseService implements Datastore { @@ -105,7 +107,10 @@ static class ReadWriteTransactionCallable implements Callable { private volatile Transaction transaction; ReadWriteTransactionCallable( - Datastore datastore, TransactionCallable callable, TransactionOptions options) { + Datastore datastore, + TransactionCallable callable, + TransactionOptions options, + @Nonnull Context parentTraceContext) { this.datastore = datastore; this.callable = callable; this.options = options; @@ -132,8 +137,14 @@ void setPrevTransactionId(ByteString transactionId) { @Override public T call() throws DatastoreException { - transaction = datastore.newTransaction(options); - try { + com.google.cloud.datastore.telemetry.TraceUtil traceUtil = + datastore.getOptions().getTraceUtil(); + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + traceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN, + datastore.getOptions().getTraceUtil().getCurrentContext()); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + transaction = datastore.newTransaction(options); T value = callable.run(transaction); transaction.commit(); return value; @@ -154,36 +165,42 @@ public T call() throws DatastoreException { @Override public T runInTransaction(final TransactionCallable callable) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_TRANSACTION); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( - new ReadWriteTransactionCallable(this, callable, null), + new ReadWriteTransactionCallable( + this, callable, null, otelTraceUtil.getCurrentContext()), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } @Override public T runInTransaction( final TransactionCallable callable, TransactionOptions transactionOptions) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_TRANSACTION); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( - new ReadWriteTransactionCallable(this, callable, transactionOptions), + new ReadWriteTransactionCallable( + this, callable, transactionOptions, otelTraceUtil.getCurrentContext()), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } @@ -634,10 +651,14 @@ private com.google.datastore.v1.CommitResponse commitMutation( com.google.datastore.v1.CommitResponse commit( final com.google.datastore.v1.CommitRequest requestPb) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT); - span.setAttribute("isTransactional", requestPb.hasTransaction()); - + final boolean isTransactional = + requestPb.hasTransaction() || requestPb.hasSingleUseTransaction(); + final String spanName = + isTransactional + ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT + : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; + com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); + span.setAttribute("isTransactional", isTransactional); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.commit(requestPb), @@ -663,7 +684,8 @@ com.google.datastore.v1.BeginTransactionResponse beginTransaction( final com.google.datastore.v1.BeginTransactionRequest requestPb) { com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION); + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION, + otelTraceUtil.getCurrentContext()); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope scope = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.beginTransaction(requestPb), diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java index f08a908ec..e730db81f 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TransactionImpl.java @@ -20,6 +20,7 @@ import com.google.api.core.BetaApi; import com.google.cloud.datastore.models.ExplainOptions; +import com.google.cloud.datastore.telemetry.TraceUtil; import com.google.common.collect.ImmutableList; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.TransactionOptions; @@ -28,6 +29,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import javax.annotation.Nonnull; final class TransactionImpl extends BaseDatastoreBatchWriter implements Transaction { @@ -37,6 +39,8 @@ final class TransactionImpl extends BaseDatastoreBatchWriter implements Transact private final ReadOptionProtoPreparer readOptionProtoPreparer; + @Nonnull private final TraceUtil traceUtil; + static class ResponseImpl implements Transaction.Response { private final com.google.datastore.v1.CommitResponse response; @@ -78,6 +82,7 @@ public List getGeneratedKeys() { transactionId = datastore.requestTransactionId(requestPb); this.readOptionProtoPreparer = new ReadOptionProtoPreparer(); + this.traceUtil = datastore.getOptions().getTraceUtil(); } @Override diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java index a4f25813a..2a42081a1 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java @@ -102,13 +102,13 @@ public TraceUtil.Span startSpan(String spanName, TraceUtil.Context parent) { @Nonnull @Override - public TraceUtil.Span currentSpan() { + public TraceUtil.Span getCurrentSpan() { return new Span(); } @Nonnull @Override - public TraceUtil.Context currentContext() { + public TraceUtil.Context getCurrentContext() { return new Context(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java index 3bf6a7466..50f89369a 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java @@ -300,13 +300,13 @@ public TraceUtil.Span startSpan(String spanName, TraceUtil.Context parent) { @Nonnull @Override - public TraceUtil.Span currentSpan() { + public TraceUtil.Span getCurrentSpan() { return new Span(io.opentelemetry.api.trace.Span.current(), ""); } @Nonnull @Override - public TraceUtil.Context currentContext() { + public TraceUtil.Context getCurrentContext() { return new Context(io.opentelemetry.context.Context.current()); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index d43c36a07..8f45e2b62 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -35,6 +35,7 @@ public interface TraceUtil { static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; + static final String SPAN_NAME_TRANSACTION_RUN = "Transaction.run"; static final String SPAN_NAME_BEGIN_TRANSACTION = "Transaction.Begin"; static final String SPAN_NAME_TRANSACTION_LOOKUP = "Transaction.Lookup"; static final String SPAN_NAME_TRANSACTION_COMMIT = "Transaction.Commit"; @@ -133,9 +134,9 @@ interface Scope extends AutoCloseable { /** Returns the current span. */ @Nonnull - Span currentSpan(); + Span getCurrentSpan(); /** Returns the current Context. */ @Nonnull - Context currentContext(); + Context getCurrentContext(); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 67df8c44b..d2cffc5db 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -22,7 +22,9 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN; import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; @@ -749,6 +751,7 @@ public void transactionalLookupTest() throws Exception { try (Scope ignored = rootSpan.makeCurrent()) { Transaction transaction = datastore.newTransaction(); Entity entity = datastore.get(KEY1, ReadOption.transactionId(transaction.getTransactionId())); + transaction.commit(); assertNull(entity); } finally { rootSpan.end(); @@ -757,18 +760,55 @@ public void transactionalLookupTest() throws Exception { fetchAndValidateTrace( customSpanContext.getTraceId(), - /*numExpectedSpans=*/ 2, - Collections.singletonList(Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION))); + /*numExpectedSpans=*/ 3, + Arrays.asList( + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP), + Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); + } + + @Test + public void runInTransactionQueryTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + Datastore.TransactionCallable callable = + transaction -> { + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + return true; + }; + datastore.runInTransaction(callable); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); fetchAndValidateTrace( customSpanContext.getTraceId(), - /*numExpectedSpans=*/ 2, - Collections.singletonList(Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP))); + /*numExpectedSpans=*/ 4, + Arrays.asList( + Collections.singletonList(SPAN_NAME_TRANSACTION_RUN), + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_RUN_QUERY), + Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); } - @Test - public void runInTransactionQueryTest() throws Exception {} - @Test public void runInTransactionAggregationQueryTest() throws Exception {} diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java index 0f3f183cd..a24f55597 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java @@ -29,16 +29,16 @@ public void disabledTraceUtilDoesNotProvideChannelConfigurator() { @Test public void usesDisabledContext() { DisabledTraceUtil traceUtil = new DisabledTraceUtil(); - assertThat(traceUtil.currentContext() instanceof DisabledTraceUtil.Context).isTrue(); + assertThat(traceUtil.getCurrentContext() instanceof DisabledTraceUtil.Context).isTrue(); } @Test public void usesDisabledSpan() { DisabledTraceUtil traceUtil = new DisabledTraceUtil(); - assertThat(traceUtil.currentSpan() instanceof DisabledTraceUtil.Span).isTrue(); + assertThat(traceUtil.getCurrentSpan() instanceof DisabledTraceUtil.Span).isTrue(); assertThat(traceUtil.startSpan("foo") instanceof DisabledTraceUtil.Span).isTrue(); assertThat( - traceUtil.startSpan("foo", traceUtil.currentContext()) + traceUtil.startSpan("foo", traceUtil.getCurrentContext()) instanceof DisabledTraceUtil.Span) .isTrue(); } @@ -46,8 +46,9 @@ public void usesDisabledSpan() { @Test public void usesDisabledScope() { DisabledTraceUtil traceUtil = new DisabledTraceUtil(); - assertThat(traceUtil.currentContext().makeCurrent() instanceof DisabledTraceUtil.Scope) + assertThat(traceUtil.getCurrentContext().makeCurrent() instanceof DisabledTraceUtil.Scope) + .isTrue(); + assertThat(traceUtil.getCurrentSpan().makeCurrent() instanceof DisabledTraceUtil.Scope) .isTrue(); - assertThat(traceUtil.currentSpan().makeCurrent() instanceof DisabledTraceUtil.Scope).isTrue(); } } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java index e88e1a849..2497672d9 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java @@ -82,23 +82,26 @@ public void enabledTraceUtilProvidesChannelConfigurator() { @Test public void usesEnabledContext() { - assertThat(newEnabledTraceUtil().currentContext() instanceof EnabledTraceUtil.Context).isTrue(); + assertThat(newEnabledTraceUtil().getCurrentContext() instanceof EnabledTraceUtil.Context) + .isTrue(); } @Test public void usesEnabledSpan() { EnabledTraceUtil traceUtil = newEnabledTraceUtil(); - assertThat(traceUtil.currentSpan() instanceof EnabledTraceUtil.Span).isTrue(); + assertThat(traceUtil.getCurrentSpan() instanceof EnabledTraceUtil.Span).isTrue(); assertThat(traceUtil.startSpan("foo") != null).isTrue(); assertThat( - traceUtil.startSpan("foo", traceUtil.currentContext()) instanceof EnabledTraceUtil.Span) + traceUtil.startSpan("foo", traceUtil.getCurrentContext()) + instanceof EnabledTraceUtil.Span) .isTrue(); } @Test public void usesEnabledScope() { EnabledTraceUtil traceUtil = newEnabledTraceUtil(); - assertThat(traceUtil.currentContext().makeCurrent() instanceof EnabledTraceUtil.Scope).isTrue(); - assertThat(traceUtil.currentSpan().makeCurrent() instanceof EnabledTraceUtil.Scope).isTrue(); + assertThat(traceUtil.getCurrentContext().makeCurrent() instanceof EnabledTraceUtil.Scope) + .isTrue(); + assertThat(traceUtil.getCurrentSpan().makeCurrent() instanceof EnabledTraceUtil.Scope).isTrue(); } } From 552d04ef6da844bf01df0283688d90980f2beb9d Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Tue, 18 Jun 2024 23:06:38 -0700 Subject: [PATCH 22/41] feat: Allocateid tracing (#1488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Adding tracing for AllocateIds RPC * formatting * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot --- README.md | 106 +----------------- google-cloud-datastore/pom.xml | 2 +- .../google/cloud/datastore/DatastoreImpl.java | 10 +- .../com/google/cloud/datastore/TraceUtil.java | 1 - .../cloud/datastore/telemetry/TraceUtil.java | 1 + .../cloud/datastore/it/ITE2ETracingTest.java | 28 +++++ 6 files changed, 41 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 9089655ec..4520f2779 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.41.0 + 26.40.0 pom import @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.42.0') +implementation platform('com.google.cloud:libraries-bom:26.40.0') implementation 'com.google.cloud:google-cloud-datastore' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-datastore:2.20.2' +implementation 'com.google.cloud:google-cloud-datastore:2.20.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.20.2" +libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.20.0" ``` @@ -210,102 +210,6 @@ running on Compute Engine or from your own desktop. To run the example on App En the code from the main method to your application's servlet class and change the print statements to display on your webpage. -gRPC Java Datastore Client User Guide -------- -In this feature launch, the [Java Datastore client](https://github.com/googleapis/java-datastore) now offers gRPC as a transport layer option with experimental support. Using [gRPC connection pooling](https://grpc.io/docs/guides/performance/) enables distributing RPCs over multiple connections which may improve performance. - -#### Download Instructions -Instructions: -1. Clone the grpc-experimental branch from GitHub: -```python -git clone -b grpc-experimental https://github.com/googleapis/java-datastore.git -``` -2. Run the following commands to build the library: -```python -# Go to the directory the code was downloaded to -cd java-datastore/ - -# Build the library -mvn clean install -DskipTests=true -``` -3. Add the following dependency to your project: -```xml - - com.google.cloud - google-cloud-datastore - 2.20.0-grpc-experimental-1-SNAPSHOT - -``` - -#### How to Use -To opt-in to the gRPC transport behavior, simply add the below line of code (`setTransportOptions`) to your Datastore client instantiation. - -Example: -```java -DatastoreOptions datastoreOptions = - DatastoreOptions.newBuilder() - .setProjectId("my-project") - .setDatabaseId("my-database") - .setTransportOptions(GrpcTransportOptions.newBuilder().build()) - .build(); -``` -Setting the transport options explicitly to `GrpcTransportOptions` will signal the client to use gRPC instead of HTTP when making calls to the server. - -To revert back to the existing stable behavior and transport, simply remove the transport options line or replace it with `HttpTransportOptions`. Please note this will require an application rebuild and restart. -Example: -```java -// will default to existing HTTP transport behavior -DatastoreOptions datastoreOptions = DatastoreOptions.newBuilder() - .setProjectId("my-project") - .setDatabaseId("my-database") - .build(); - -// will also default to existing HTTP transport behavior -DatastoreOptions datastoreOptions = - DatastoreOptions.newBuilder() - .setProjectId("my-project") - .setDatabaseId("my-database") - .setTransportOptions(HttpTransportOptions.newBuilder() - .setConnectTimeout(1000) - .build()).build(); -``` - -Note: client instantiations that already use `setTransportOptions` with `HttpTransportOptions` will continue to have the same behavior. Only transports that are explicitly set to gRPC will change. - -#### Verify Datastore Transport Options Type -To verify which type of TransportOptions you have successfully configured, you can use the below lines of code to compare transport options type: -```java -// checks if using gRPC transport options -boolean isGRPC = datastore.getOptions().getTransportOptions() instanceof GrpcTransportOptions; - -// checks if using HTTP transport options -boolean isHTTP = datastore.getOptions().getTransportOptions() instanceof HTTPTransportOptions; -``` - -#### New Features -There are new gRPC specific features available to use in this update. - -##### Channel Pooling -To customize the number of channels your client uses, you can update the channel provider in the DatastoreOptions. -See [ChannelPoolSettings](https://cloud.google.com/java/docs/reference/gax/latest/com.google.api.gax.grpc.ChannelPoolSettings) and [Performance Best Practices](https://grpc.io/docs/guides/performance/) for more information on channel pools and best practices for performance. - -Example: -```java -InstantiatingGrpcChannelProvider channelProvider = - DatastoreSettings.defaultGrpcTransportProviderBuilder() - .setChannelPoolSettings( - ChannelPoolSettings.builder() - .setInitialChannelCount(MIN_VAL) - .setMaxChannelCount(MAX_VAL) - .build()) - .build(); - -DatastoreOptions options = DatastoreOptions.newBuilder() - .setProjectId("my-project") - .setChannelProvider(channelProvider) - .setTransportOptions(GrpcTransportOptions.newBuilder().build()) - .build(); -``` Testing ------- @@ -480,7 +384,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-datastore/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-datastore.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.20.2 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.20.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 478c6f785..51b8a07b4 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -16,7 +16,7 @@ google-cloud-datastore - 1.37.0 + 1.38.0 diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index eea94cc60..f726f0bb2 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -327,8 +327,10 @@ public List allocateId(IncompleteKey... keys) { private com.google.datastore.v1.AllocateIdsResponse allocateIds( final com.google.datastore.v1.AllocateIdsRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_ALLOCATEIDS); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_ALLOCATE_IDS); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( new Callable() { @Override @@ -340,10 +342,10 @@ public com.google.datastore.v1.AllocateIdsResponse call() throws DatastoreExcept EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java index 57525d15d..6b51d6a87 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java @@ -31,7 +31,6 @@ public class TraceUtil { private final Tracer tracer = Tracing.getTracer(); private static final TraceUtil traceUtil = new TraceUtil(); - static final String SPAN_NAME_ALLOCATEIDS = "CloudDatastoreOperation.allocateIds"; static final String SPAN_NAME_TRANSACTION = "CloudDatastoreOperation.readWriteTransaction"; static final String SPAN_NAME_BEGINTRANSACTION = "CloudDatastoreOperation.beginTransaction"; static final String SPAN_NAME_COMMIT = "CloudDatastoreOperation.commit"; diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 8f45e2b62..1198460a4 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -32,6 +32,7 @@ public interface TraceUtil { static final String ENABLE_TRACING_ENV_VAR = "DATASTORE_ENABLE_TRACING"; static final String LIBRARY_NAME = "com.google.cloud.datastore"; static final String SPAN_NAME_LOOKUP = "Lookup"; + static final String SPAN_NAME_ALLOCATE_IDS = "AllocateIds"; static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index d2cffc5db..a032762c8 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -17,6 +17,7 @@ package com.google.cloud.datastore.it; import static com.google.cloud.datastore.aggregation.Aggregation.count; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_ALLOCATE_IDS; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; @@ -40,7 +41,9 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.IncompleteKey; import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.KeyFactory; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.ReadOption; @@ -574,6 +577,31 @@ public void lookupTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_LOOKUP); } + @Test + public void allocateIdsTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + String kind1 = "kind1"; + KeyFactory keyFactory = datastore.newKeyFactory().setKind(kind1); + IncompleteKey pk1 = keyFactory.newKey(); + Key key1 = datastore.allocateId(pk1); + assertEquals(key1.getProjectId(), pk1.getProjectId()); + assertEquals(key1.getNamespace(), pk1.getNamespace()); + assertEquals(key1.getAncestors(), pk1.getAncestors()); + assertEquals(key1.getKind(), pk1.getKind()); + assertTrue(key1.hasId()); + assertFalse(key1.hasName()); + assertEquals(Key.newBuilder(pk1, key1.getId()).build(), key1); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_ALLOCATE_IDS); + } + @Test public void commitTraceTest() throws Exception { assertNotNull(customSpanContext); From 975464ac6fb8166d5d4b7d5b84c5d311c36a1126 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Tue, 25 Jun 2024 13:42:55 -0700 Subject: [PATCH 23/41] feat: Add tracing for ReserveIds operation (#1490) - added end-to-end test --- .../google/cloud/datastore/DatastoreImpl.java | 10 ++++++---- .../cloud/datastore/telemetry/TraceUtil.java | 1 + .../cloud/datastore/it/ITE2ETracingTest.java | 20 +++++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index f726f0bb2..52acb77d8 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -543,8 +543,10 @@ public List reserveIds(Key... keys) { com.google.datastore.v1.ReserveIdsResponse reserveIds( final com.google.datastore.v1.ReserveIdsRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_RESERVEIDS); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RESERVE_IDS); + try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( new Callable() { @Override @@ -556,10 +558,10 @@ public com.google.datastore.v1.ReserveIdsResponse call() throws DatastoreExcepti EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 1198460a4..1e5226126 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -33,6 +33,7 @@ public interface TraceUtil { static final String LIBRARY_NAME = "com.google.cloud.datastore"; static final String SPAN_NAME_LOOKUP = "Lookup"; static final String SPAN_NAME_ALLOCATE_IDS = "AllocateIds"; + static final String SPAN_NAME_RESERVE_IDS = "ReserveIds"; static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index a032762c8..bf7635266 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -21,6 +21,7 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RESERVE_IDS; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT; @@ -602,6 +603,25 @@ public void allocateIdsTraceTest() throws Exception { fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_ALLOCATE_IDS); } + @Test + public void reserveIdsTraceTest() throws Exception { + assertNotNull(customSpanContext); + + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + KeyFactory keyFactory = datastore.newKeyFactory().setKind("MyKind"); + Key key1 = keyFactory.newKey(10); + Key key2 = keyFactory.newKey("name"); + List keyList = datastore.reserveIds(key1, key2); + assertEquals(2, keyList.size()); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace(customSpanContext.getTraceId(), SPAN_NAME_RESERVE_IDS); + } + @Test public void commitTraceTest() throws Exception { assertNotNull(customSpanContext); From 27caf81f0398cd096498f0f06b66a3b0f2b2af3a Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Thu, 27 Jun 2024 11:51:17 -0700 Subject: [PATCH 24/41] fix: Fixed Span nesting for `ReadWriteTransactionCallable` by using parent SpanContext instead of just parent Context (#1495) * fix: Fixed the TraceUtil.startSpan method to use `SpanContext` for linking with the parent instead of `Context`. - This fixes the hierarchy of Spans appearing in a transaction under a Run method. - Tested using existing transaction test * Fixed commit reordering and typos * fix: lint errors * fix: Refactored the ReadWriteTransactioncallable.call method to use startSpan idiomatically - TraceUtil.startSpan needs more debugging - return DefaultTracerProvider instance (no-op) when initializing DisabledTraceUtil - this fixes the unit tests in DatastoreTest.testRunInTransactionWithReadWriteOption * feat: Added tracing for Transaction.RunQuery (#1499) * feat: Added span for Transactional RunQuery - tested * fix: lint * fix: patch apply issues * fix: refactor using boolean flag * fix: s/startSpan/startSpanWithParentContext --- .../google/cloud/datastore/DatastoreImpl.java | 92 ++++++++++++------- .../telemetry/DisabledTraceUtil.java | 28 +++++- .../datastore/telemetry/EnabledTraceUtil.java | 39 +++++++- .../cloud/datastore/telemetry/TraceUtil.java | 25 ++++- .../cloud/datastore/it/ITE2ETracingTest.java | 51 +++++++++- .../telemetry/DisabledTraceUtilTest.java | 2 +- .../telemetry/EnabledTraceUtilTest.java | 2 +- 7 files changed, 193 insertions(+), 46 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 52acb77d8..0fefc3168 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -25,9 +25,9 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.datastore.execution.AggregationQueryExecutor; import com.google.cloud.datastore.spi.v1.DatastoreRpc; -import com.google.cloud.datastore.telemetry.TraceUtil.Context; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -42,6 +42,11 @@ import io.opencensus.common.Scope; import io.opencensus.trace.Span; import io.opencensus.trace.Status; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Context; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -53,7 +58,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; -import javax.annotation.Nonnull; +import javax.annotation.Nullable; final class DatastoreImpl extends BaseService implements Datastore { @@ -106,15 +111,18 @@ static class ReadWriteTransactionCallable implements Callable { private volatile TransactionOptions options; private volatile Transaction transaction; + private final com.google.cloud.datastore.telemetry.TraceUtil.SpanContext parentSpanContext; + ReadWriteTransactionCallable( Datastore datastore, TransactionCallable callable, TransactionOptions options, - @Nonnull Context parentTraceContext) { + @Nullable com.google.cloud.datastore.telemetry.TraceUtil.SpanContext parentSpanContext) { this.datastore = datastore; this.callable = callable; this.options = options; this.transaction = null; + this.parentSpanContext = parentSpanContext; } Datastore getDatastore() { @@ -135,26 +143,57 @@ void setPrevTransactionId(ByteString transactionId) { options = options.toBuilder().setReadWrite(readWrite).build(); } + private io.opentelemetry.api.trace.Span startSpanWithParentContext( + String spanName, + com.google.cloud.datastore.telemetry.TraceUtil.SpanContext parentSpanContext) { + com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil = + datastore.getOptions().getTraceUtil(); + SpanBuilder spanBuilder = + otelTraceUtil + .getTracer() + .spanBuilder(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN) + .setSpanKind(SpanKind.PRODUCER) + .setParent( + Context.current() + .with( + io.opentelemetry.api.trace.Span.wrap( + parentSpanContext.getSpanContext()))); + return spanBuilder.startSpan(); + } + @Override public T call() throws DatastoreException { - com.google.cloud.datastore.telemetry.TraceUtil traceUtil = - datastore.getOptions().getTraceUtil(); - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - traceUtil.startSpan( + // TODO Instead of using OTel Spans directly, TraceUtil.Span should be used here. However, + // the same code in startSpanInternal doesn't work when EnabledTraceUtil.StartSpan is called + // probably because of some thread-local caching that is getting lost. This needs more + // debugging. The code below works and is idiomatic but could be prettier and more consistent + // with the use of TraceUtil-provided framework. + io.opentelemetry.api.trace.Span span = + startSpanWithParentContext( com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN, - datastore.getOptions().getTraceUtil().getCurrentContext()); - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + parentSpanContext); + try (io.opentelemetry.context.Scope ignored = span.makeCurrent()) { transaction = datastore.newTransaction(options); T value = callable.run(transaction); transaction.commit(); return value; } catch (Exception ex) { transaction.rollback(); + span.setStatus(StatusCode.ERROR, ex.getMessage()); + span.recordException( + ex, + Attributes.builder() + .put("exception.message", ex.getMessage()) + .put("exception.type", ex.getClass().getName()) + .put("exception.stacktrace", Throwables.getStackTraceAsString(ex)) + .build()); + span.end(); throw DatastoreException.propagateUserException(ex); } finally { if (transaction.isActive()) { transaction.rollback(); } + span.end(); if (options != null && options.getModeCase().equals(TransactionOptions.ModeCase.READ_WRITE)) { setPrevTransactionId(transaction.getTransactionId()); @@ -165,42 +204,30 @@ public T call() throws DatastoreException { @Override public T runInTransaction(final TransactionCallable callable) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + try { return RetryHelper.runWithRetries( new ReadWriteTransactionCallable( - this, callable, null, otelTraceUtil.getCurrentContext()), + this, callable, null, otelTraceUtil.getCurrentSpanContext()), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.end(e); throw DatastoreException.translateAndThrow(e); - } finally { - span.end(); } } @Override public T runInTransaction( final TransactionCallable callable, TransactionOptions transactionOptions) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN); - try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { + try { return RetryHelper.runWithRetries( new ReadWriteTransactionCallable( - this, callable, transactionOptions, otelTraceUtil.getCurrentContext()), + this, callable, transactionOptions, otelTraceUtil.getCurrentSpanContext()), retrySettings, TRANSACTION_EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.end(e); throw DatastoreException.translateAndThrow(e); - } finally { - span.end(); } } @@ -258,11 +285,14 @@ public AggregationResults runAggregation( com.google.datastore.v1.RunQueryResponse runQuery( final com.google.datastore.v1.RunQueryRequest requestPb) { - com.google.cloud.datastore.telemetry.TraceUtil.Span span = - otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY); ReadOptions readOptions = requestPb.getReadOptions(); - span.setAttribute( - "isTransactional", readOptions.hasTransaction() || readOptions.hasNewTransaction()); + boolean isTransactional = readOptions.hasTransaction() || readOptions.hasNewTransaction(); + String spanName = + (isTransactional + ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN_QUERY + : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY); + com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); + span.setAttribute("isTransactional", isTransactional); span.setAttribute("readConsistency", readOptions.getReadConsistency().toString()); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { @@ -275,7 +305,7 @@ com.google.datastore.v1.RunQueryResponse runQuery( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); span.addEvent( - com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY + ": Completed", + spanName + ": Completed", new ImmutableMap.Builder() .put("Received", response.getBatch().getEntityResultsCount()) .put("More results", response.getBatch().getMoreResults().toString()) @@ -689,7 +719,7 @@ com.google.datastore.v1.BeginTransactionResponse beginTransaction( com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan( com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_BEGIN_TRANSACTION, - otelTraceUtil.getCurrentContext()); + otelTraceUtil.getCurrentSpanContext()); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope scope = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> datastoreRpc.beginTransaction(requestPb), diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java index 2a42081a1..6ba0fd81c 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java @@ -19,7 +19,11 @@ import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; import com.google.api.core.InternalApi; +import com.google.cloud.datastore.telemetry.TraceUtil.SpanContext; import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -31,6 +35,13 @@ @InternalApi public class DisabledTraceUtil implements TraceUtil { + static class SpanContext implements TraceUtil.SpanContext { + @Override + public io.opentelemetry.api.trace.SpanContext getSpanContext() { + return null; + } + } + static class Span implements TraceUtil.Span { @Override public void end() {} @@ -66,6 +77,10 @@ public TraceUtil.Span setAttribute(String key, boolean value) { return this; } + public io.opentelemetry.api.trace.Span getSpan() { + return null; + } + @Override public Scope makeCurrent() { return new Scope(); @@ -96,7 +111,7 @@ public Span startSpan(String spanName) { } @Override - public TraceUtil.Span startSpan(String spanName, TraceUtil.Context parent) { + public TraceUtil.Span startSpan(String spanName, TraceUtil.SpanContext parentSpanContext) { return new Span(); } @@ -111,4 +126,15 @@ public TraceUtil.Span getCurrentSpan() { public TraceUtil.Context getCurrentContext() { return new Context(); } + + @Nonnull + @Override + public TraceUtil.SpanContext getCurrentSpanContext() { + return new SpanContext(); + } + + @Override + public Tracer getTracer() { + return TracerProvider.noop().get(LIBRARY_NAME); + } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java index 50f89369a..438395cb1 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java @@ -22,16 +22,19 @@ import com.google.api.core.ApiFutures; import com.google.api.core.InternalApi; import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.telemetry.TraceUtil.SpanContext; import com.google.common.base.Throwables; import io.grpc.ManagedChannelBuilder; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -70,6 +73,19 @@ public ApiFunction getChannelConfi return null; } + static class SpanContext implements TraceUtil.SpanContext { + private final io.opentelemetry.api.trace.SpanContext spanContext; + + public SpanContext(io.opentelemetry.api.trace.SpanContext spanContext) { + this.spanContext = spanContext; + } + + @Override + public io.opentelemetry.api.trace.SpanContext getSpanContext() { + return this.spanContext; + } + } + static class Span implements TraceUtil.Span { private final io.opentelemetry.api.trace.Span span; private final String spanName; @@ -181,6 +197,10 @@ public TraceUtil.Span setAttribute(String key, boolean value) { return this; } + public io.opentelemetry.api.trace.Span getSpan() { + return this.span; + } + @Override public Scope makeCurrent() { try (io.opentelemetry.context.Scope scope = span.makeCurrent()) { @@ -286,13 +306,15 @@ public Span startSpan(String spanName) { } @Override - public TraceUtil.Span startSpan(String spanName, TraceUtil.Context parent) { - assert (parent instanceof Context); + public TraceUtil.Span startSpan(String spanName, TraceUtil.SpanContext parentSpanContext) { SpanBuilder spanBuilder = tracer .spanBuilder(spanName) .setSpanKind(SpanKind.PRODUCER) - .setParent(((Context) parent).context); + .setParent( + io.opentelemetry.context.Context.current() + .with( + io.opentelemetry.api.trace.Span.wrap(parentSpanContext.getSpanContext()))); io.opentelemetry.api.trace.Span span = addSettingsAttributesToCurrentSpan(spanBuilder).startSpan(); return new Span(span, spanName); @@ -309,4 +331,15 @@ public TraceUtil.Span getCurrentSpan() { public TraceUtil.Context getCurrentContext() { return new Context(io.opentelemetry.context.Context.current()); } + + @Nonnull + @Override + public TraceUtil.SpanContext getCurrentSpanContext() { + return new SpanContext(io.opentelemetry.api.trace.Span.current().getSpanContext()); + } + + @Override + public Tracer getTracer() { + return this.tracer; + } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index 1e5226126..dd1dcf29e 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -21,6 +21,8 @@ import com.google.api.core.InternalExtensionOnly; import com.google.cloud.datastore.DatastoreOptions; import io.grpc.ManagedChannelBuilder; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; import java.util.Map; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -37,10 +39,11 @@ public interface TraceUtil { static final String SPAN_NAME_COMMIT = "Commit"; static final String SPAN_NAME_RUN_QUERY = "RunQuery"; static final String SPAN_NAME_RUN_AGGREGATION_QUERY = "RunAggregationQuery"; - static final String SPAN_NAME_TRANSACTION_RUN = "Transaction.run"; + static final String SPAN_NAME_TRANSACTION_RUN = "Transaction.Run"; static final String SPAN_NAME_BEGIN_TRANSACTION = "Transaction.Begin"; static final String SPAN_NAME_TRANSACTION_LOOKUP = "Transaction.Lookup"; static final String SPAN_NAME_TRANSACTION_COMMIT = "Transaction.Commit"; + static final String SPAN_NAME_TRANSACTION_RUN_QUERY = "Transaction.RunQuery"; static final String SPAN_NAME_ROLLBACK = "Transaction.Rollback"; static final String SPAN_NAME_TRANSACTION_RUN_AGGREGATION_QUERY = "Transaction.RunAggregationQuery"; @@ -78,6 +81,11 @@ static TraceUtil getInstance(@Nonnull DatastoreOptions datastoreOptions) { @Nullable ApiFunction getChannelConfigurator(); + /** Represents a trace span's context */ + interface SpanContext { + io.opentelemetry.api.trace.SpanContext getSpanContext(); + } + /** Represents a trace span. */ interface Span { /** Adds the given event to this span. */ @@ -95,6 +103,8 @@ interface Span { /** Adds the given attribute to this span. */ Span setAttribute(String key, boolean value); + io.opentelemetry.api.trace.Span getSpan(); + /** Marks this span as the current span. */ Scope makeCurrent(); @@ -129,10 +139,10 @@ interface Scope extends AutoCloseable { Span startSpan(String spanName); /** - * Starts a new span with the given name and the given context as its parent, sets it as the - * current span, and returns it. + * Starts a new span with the given name and the span represented by the parentSpanContext as its + * parents, sets it as the current span and returns it. */ - Span startSpan(String spanName, Context parent); + Span startSpan(String spanName, SpanContext parentSpanContext); /** Returns the current span. */ @Nonnull @@ -141,4 +151,11 @@ interface Scope extends AutoCloseable { /** Returns the current Context. */ @Nonnull Context getCurrentContext(); + + /** Returns the current SpanContext */ + @Nonnull + SpanContext getCurrentSpanContext(); + + /** Returns the current OpenTelemetry Tracer when OpenTelemetry SDK is provided. */ + Tracer getTracer(); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index bf7635266..34c7ea858 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -27,6 +27,7 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN_QUERY; import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; @@ -815,8 +816,49 @@ public void transactionalLookupTest() throws Exception { Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); } + @Test + public void transactionQueryTest() throws Exception { + // Set up + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + assertNotNull(customSpanContext); + + // Test + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + Transaction transaction = datastore.newTransaction(); + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + QueryResults queryResults = transaction.run(query); + transaction.commit(); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + } finally { + rootSpan.end(); + } + waitForTracesToComplete(); + + fetchAndValidateTrace( + customSpanContext.getTraceId(), + /*numExpectedSpans=*/ 3, + Arrays.asList( + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_TRANSACTION_RUN_QUERY), + Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); + } + @Test public void runInTransactionQueryTest() throws Exception { + // Set up Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); List entityList = new ArrayList<>(); @@ -851,14 +893,13 @@ public void runInTransactionQueryTest() throws Exception { customSpanContext.getTraceId(), /*numExpectedSpans=*/ 4, Arrays.asList( - Collections.singletonList(SPAN_NAME_TRANSACTION_RUN), - Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), - Collections.singletonList(SPAN_NAME_RUN_QUERY), - Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); + Arrays.asList(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_BEGIN_TRANSACTION), + Arrays.asList(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_RUN_QUERY), + Arrays.asList(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_TRANSACTION_COMMIT))); } @Test - public void runInTransactionAggregationQueryTest() throws Exception {} + public void transactionRunQueryTest() throws Exception {} @Test public void readWriteTransactionTraceTest() throws Exception {} diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java index a24f55597..89c91b3a7 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DisabledTraceUtilTest.java @@ -38,7 +38,7 @@ public void usesDisabledSpan() { assertThat(traceUtil.getCurrentSpan() instanceof DisabledTraceUtil.Span).isTrue(); assertThat(traceUtil.startSpan("foo") instanceof DisabledTraceUtil.Span).isTrue(); assertThat( - traceUtil.startSpan("foo", traceUtil.getCurrentContext()) + traceUtil.startSpan("foo", traceUtil.getCurrentSpanContext()) instanceof DisabledTraceUtil.Span) .isTrue(); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java index 2497672d9..a3620bbc2 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/EnabledTraceUtilTest.java @@ -92,7 +92,7 @@ public void usesEnabledSpan() { assertThat(traceUtil.getCurrentSpan() instanceof EnabledTraceUtil.Span).isTrue(); assertThat(traceUtil.startSpan("foo") != null).isTrue(); assertThat( - traceUtil.startSpan("foo", traceUtil.getCurrentContext()) + traceUtil.startSpan("foo", traceUtil.getCurrentSpanContext()) instanceof EnabledTraceUtil.Span) .isTrue(); } From 31a6f4a39a66f51f35490315bdb0ee52204c16c3 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Tue, 2 Jul 2024 12:56:52 -0700 Subject: [PATCH 25/41] test: Additional Transaction Testing and cleanup OpenCensus usage (#1505) * test: newTransactionReadWriteTraceTest * fix: test literal * feat: Added tests for transaction cases * fix: Cleanup OpenCensus dead code * fix: updating version from 2.20.1 -> 2.21.0 * fix: reverting version from 2.21.0 -> 2.20.1 * fix: Adding an exception to the clirr-maven-plugin for an internal API parameter change from com.google.cloud.datastore.TraceUtil -> com.google.cloud.datastore.telemetry.TraceUtil * fix: Fixing the differenceType in clirr exception * fix: add an exception for removal of an internal class (com.google.cloud.datastore.TraceUtil) * fix: fixing incomplete difference details for type 7005 * fix: Fixing `to` of the difference to be the entire signature * fix: typo --- .../clirr-ignored-differences.xml | 12 ++ .../google/cloud/datastore/DatastoreImpl.java | 17 ++- .../RetryAndTraceDatastoreRpcDecorator.java | 5 +- .../com/google/cloud/datastore/TraceUtil.java | 75 ----------- .../datastore/spi/v1/HttpDatastoreRpc.java | 5 +- .../cloud/datastore/it/ITE2ETracingTest.java | 122 ++++++++++++++++-- 6 files changed, 139 insertions(+), 97 deletions(-) delete mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java diff --git a/google-cloud-datastore/clirr-ignored-differences.xml b/google-cloud-datastore/clirr-ignored-differences.xml index 1620fd752..847ba4f4c 100644 --- a/google-cloud-datastore/clirr-ignored-differences.xml +++ b/google-cloud-datastore/clirr-ignored-differences.xml @@ -55,4 +55,16 @@ java.lang.Object execute(com.google.cloud.datastore.Query, com.google.cloud.datastore.ReadOption[]) 7004 + + com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator + RetryAndTraceDatastoreRpcDecorator(com.google.cloud.datastore.spi.v1.DatastoreRpc, com.google.cloud.datastore.TraceUtil, com.google.api.gax.retrying.RetrySettings, com.google.cloud.datastore.DatastoreOptions) + RetryAndTraceDatastoreRpcDecorator(com.google.cloud.datastore.spi.v1.DatastoreRpc, com.google.cloud.datastore.telemetry.TraceUtil, com.google.api.gax.retrying.RetrySettings, com.google.cloud.datastore.DatastoreOptions) + 7005 + + + + + com/google/cloud/datastore/TraceUtil + 8001 + diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 0fefc3168..ea85170fe 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -25,6 +25,7 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.datastore.execution.AggregationQueryExecutor; import com.google.cloud.datastore.spi.v1.DatastoreRpc; +import com.google.cloud.datastore.telemetry.TraceUtil.Scope; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; @@ -39,9 +40,6 @@ import com.google.datastore.v1.RunQueryResponse; import com.google.datastore.v1.TransactionOptions; import com.google.protobuf.ByteString; -import io.opencensus.common.Scope; -import io.opencensus.trace.Span; -import io.opencensus.trace.Status; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; @@ -68,7 +66,6 @@ final class DatastoreImpl extends BaseService implements Datas TransactionExceptionHandler.build(); private static final ExceptionHandler TRANSACTION_OPERATION_EXCEPTION_HANDLER = TransactionOperationExceptionHandler.build(); - private final TraceUtil traceUtil = TraceUtil.getInstance(); private final com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil = getOptions().getTraceUtil(); @@ -85,7 +82,8 @@ final class DatastoreImpl extends BaseService implements Datas readOptionProtoPreparer = new ReadOptionProtoPreparer(); aggregationQueryExecutor = new AggregationQueryExecutor( - new RetryAndTraceDatastoreRpcDecorator(datastoreRpc, traceUtil, retrySettings, options), + new RetryAndTraceDatastoreRpcDecorator( + datastoreRpc, otelTraceUtil, retrySettings, options), options); } @@ -744,8 +742,9 @@ void rollbackTransaction(ByteString transaction) { } void rollback(final com.google.datastore.v1.RollbackRequest requestPb) { - Span span = traceUtil.startSpan(TraceUtil.SPAN_NAME_ROLLBACK); - try (Scope scope = traceUtil.getTracer().withSpan(span)) { + com.google.cloud.datastore.telemetry.TraceUtil.Span span = + otelTraceUtil.startSpan(com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_ROLLBACK); + try (Scope scope = span.makeCurrent()) { RetryHelper.runWithRetries( new Callable() { @Override @@ -758,10 +757,10 @@ public Void call() throws DatastoreException { EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelperException e) { - span.setStatus(Status.UNKNOWN.withDescription(e.getMessage())); + span.end(e); throw DatastoreException.translateAndThrow(e); } finally { - span.end(TraceUtil.END_SPAN_OPTIONS); + span.end(); } } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java index 7abf1d360..e1ea6e8dd 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java @@ -22,6 +22,7 @@ import com.google.cloud.RetryHelper; import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.datastore.spi.v1.DatastoreRpc; +import com.google.cloud.datastore.telemetry.TraceUtil; import com.google.datastore.v1.AllocateIdsRequest; import com.google.datastore.v1.AllocateIdsResponse; import com.google.datastore.v1.BeginTransactionRequest; @@ -55,13 +56,13 @@ public class RetryAndTraceDatastoreRpcDecorator implements DatastoreRpc { public RetryAndTraceDatastoreRpcDecorator( DatastoreRpc datastoreRpc, - TraceUtil traceUtil, + TraceUtil otelTraceUtil, RetrySettings retrySettings, DatastoreOptions datastoreOptions) { this.datastoreRpc = datastoreRpc; this.retrySettings = retrySettings; this.datastoreOptions = datastoreOptions; - this.otelTraceUtil = datastoreOptions.getTraceUtil(); + this.otelTraceUtil = otelTraceUtil; } @Override diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java deleted file mode 100644 index 6b51d6a87..000000000 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/TraceUtil.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2020 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.datastore; - -import com.google.cloud.datastore.spi.v1.HttpDatastoreRpc; -import io.opencensus.trace.EndSpanOptions; -import io.opencensus.trace.Span; -import io.opencensus.trace.Tracer; -import io.opencensus.trace.Tracing; - -/** - * Helper class for tracing utility. It is used for instrumenting {@link HttpDatastoreRpc} with - * OpenCensus APIs. - * - *

TraceUtil instances are created by the {@link TraceUtil#getInstance()} method. - */ -public class TraceUtil { - private final Tracer tracer = Tracing.getTracer(); - private static final TraceUtil traceUtil = new TraceUtil(); - static final String SPAN_NAME_TRANSACTION = "CloudDatastoreOperation.readWriteTransaction"; - static final String SPAN_NAME_BEGINTRANSACTION = "CloudDatastoreOperation.beginTransaction"; - static final String SPAN_NAME_COMMIT = "CloudDatastoreOperation.commit"; - static final String SPAN_NAME_LOOKUP = "CloudDatastoreOperation.lookup"; - static final String SPAN_NAME_RESERVEIDS = "CloudDatastoreOperation.reserveIds"; - static final String SPAN_NAME_ROLLBACK = "CloudDatastoreOperation.rollback"; - static final String SPAN_NAME_RUNQUERY = "CloudDatastoreOperation.runQuery"; - static final String SPAN_NAME_RUN_AGGREGATION_QUERY = - "CloudDatastoreOperation.runAggregationQuery"; - static final EndSpanOptions END_SPAN_OPTIONS = - EndSpanOptions.builder().setSampleToLocalSpanStore(true).build(); - - /** - * Starts a new span. - * - * @param spanName The name of the returned Span. - * @return The newly created {@link Span}. - */ - protected Span startSpan(String spanName) { - return tracer.spanBuilder(spanName).startSpan(); - } - - /** - * Return the global {@link Tracer}. - * - * @return The global {@link Tracer}. - */ - public Tracer getTracer() { - return tracer; - } - - /** - * Return TraceUtil Object. - * - * @return An instance of {@link TraceUtil} - */ - public static TraceUtil getInstance() { - return traceUtil; - } - - private TraceUtil() {} -} diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/spi/v1/HttpDatastoreRpc.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/spi/v1/HttpDatastoreRpc.java index fd3cdc658..d927a2d7f 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/spi/v1/HttpDatastoreRpc.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/spi/v1/HttpDatastoreRpc.java @@ -21,7 +21,6 @@ import com.google.api.client.http.HttpTransport; import com.google.cloud.datastore.DatastoreException; import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.TraceUtil; import com.google.cloud.http.CensusHttpModule; import com.google.cloud.http.HttpTransportOptions; import com.google.datastore.v1.AllocateIdsRequest; @@ -40,6 +39,7 @@ import com.google.datastore.v1.RunAggregationQueryResponse; import com.google.datastore.v1.RunQueryRequest; import com.google.datastore.v1.RunQueryResponse; +import io.opencensus.trace.Tracing; import java.io.IOException; import java.net.InetAddress; import java.net.URL; @@ -80,8 +80,7 @@ public HttpDatastoreRpc(DatastoreOptions options) { private HttpRequestInitializer getHttpRequestInitializer( final DatastoreOptions options, HttpTransportOptions httpTransportOptions) { // Open Census initialization - CensusHttpModule censusHttpModule = - new CensusHttpModule(TraceUtil.getInstance().getTracer(), true); + CensusHttpModule censusHttpModule = new CensusHttpModule(Tracing.getTracer(), true); final HttpRequestInitializer censusHttpModuleHttpRequestInitializer = censusHttpModule.getHttpRequestInitializer( httpTransportOptions.getHttpRequestInitializer(options)); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 34c7ea858..b8174184e 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -22,6 +22,7 @@ import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RESERVE_IDS; +import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_ROLLBACK; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY; import static com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT; @@ -374,6 +375,7 @@ public void after() throws Exception { tracer = null; retrievedTrace = null; customSpanContext = null; + openTelemetrySdk = null; } @AfterClass @@ -793,7 +795,7 @@ public void runAggregationQueryTraceTest() throws Exception { } @Test - public void transactionalLookupTest() throws Exception { + public void newTransactionReadTest() throws Exception { assertNotNull(customSpanContext); Span rootSpan = getNewRootSpanWithContext(); @@ -817,7 +819,7 @@ public void transactionalLookupTest() throws Exception { } @Test - public void transactionQueryTest() throws Exception { + public void newTransactionQueryTest() throws Exception { // Set up Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); @@ -856,6 +858,116 @@ public void transactionQueryTest() throws Exception { Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); } + @Test + public void newTransactionReadWriteTraceTest() throws Exception { + // Set up + Entity entity1 = Entity.newBuilder(KEY1).set("pepper_type", "jalapeno").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("pepper_type", "habanero").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + String simplified_spice_level = "not_spicy"; + Entity entity1update = + Entity.newBuilder(entity1).set("spice_level", simplified_spice_level).build(); + + assertNotNull(customSpanContext); + + // Test + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + Transaction transaction = datastore.newTransaction(); + entity1 = transaction.get(KEY1); + switch (entity1.getString("pepper_type")) { + case "jalapeno": + simplified_spice_level = "mild"; + break; + + case "habanero": + simplified_spice_level = "hot"; + break; + } + transaction.update(entity1update); + transaction.delete(KEY2); + transaction.commit(); + assertFalse(transaction.isActive()); + } finally { + rootSpan.end(); + } + + waitForTracesToComplete(); + + List list = datastore.fetch(KEY1, KEY2); + assertEquals(list.get(0), entity1update); + assertNull(list.get(1)); + + fetchAndValidateTrace( + customSpanContext.getTraceId(), + /*numExpectedSpans=*/ 3, + Arrays.asList( + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP), + Collections.singletonList(SPAN_NAME_TRANSACTION_COMMIT))); + } + + @Test + public void newTransactionRollbackTest() throws Exception { + // Set up + Entity entity1 = Entity.newBuilder(KEY1).set("pepper_type", "jalapeno").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("pepper_type", "habanero").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + String simplified_spice_level = "not_spicy"; + Entity entity1update = + Entity.newBuilder(entity1).set("spice_level", simplified_spice_level).build(); + + assertNotNull(customSpanContext); + + // Test + Span rootSpan = getNewRootSpanWithContext(); + try (Scope ignored = rootSpan.makeCurrent()) { + Transaction transaction = datastore.newTransaction(); + entity1 = transaction.get(KEY1); + switch (entity1.getString("pepper_type")) { + case "jalapeno": + simplified_spice_level = "mild"; + break; + + case "habanero": + simplified_spice_level = "hot"; + break; + } + transaction.update(entity1update); + transaction.delete(KEY2); + transaction.rollback(); + assertFalse(transaction.isActive()); + } finally { + rootSpan.end(); + } + + waitForTracesToComplete(); + + List list = datastore.fetch(KEY1, KEY2); + assertEquals(list.get(0), entity1); + assertEquals(list.get(1), entity2); + + fetchAndValidateTrace( + customSpanContext.getTraceId(), + /*numExpectedSpans=*/ 3, + Arrays.asList( + Collections.singletonList(SPAN_NAME_BEGIN_TRANSACTION), + Collections.singletonList(SPAN_NAME_TRANSACTION_LOOKUP), + Collections.singletonList(SPAN_NAME_ROLLBACK))); + } + @Test public void runInTransactionQueryTest() throws Exception { // Set up @@ -897,10 +1009,4 @@ public void runInTransactionQueryTest() throws Exception { Arrays.asList(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_RUN_QUERY), Arrays.asList(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_TRANSACTION_COMMIT))); } - - @Test - public void transactionRunQueryTest() throws Exception {} - - @Test - public void readWriteTransactionTraceTest() throws Exception {} } From 536c2d90bc19bdf2e417e71897a42b55ac4f90cc Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:32:13 -0700 Subject: [PATCH 26/41] =?UTF-8?q?test:=20Adding=20ITTracingTest=20to=20ver?= =?UTF-8?q?ify=20events=20and=20span=20attributes=20(whic=E2=80=A6=20(#151?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: Adding ITTracingTest to verify events and span attributes (which are not verified in ITE2ETracingTest) due to TraceClient API limitations. - This test uses InMemorySpanExporter to read the generated Otel span data by the test process to verify generated span data as it were before exporting to a backend. None of the span data is exported to a durable backend. - This test is still an E2E test as it requires a project to send RPCs to. * fix: fixing compilation error due to missing pom dependency. * test: Test for AllocateId and ReserveId rpcs * test: Commit/Put/Update/Delete tests * test: Added fixes and test for RunQuery event --- google-cloud-datastore/pom.xml | 6 + .../google/cloud/datastore/DatastoreImpl.java | 46 +- .../cloud/datastore/it/ITE2ETracingTest.java | 4 +- .../cloud/datastore/it/ITTracingTest.java | 582 ++++++++++++++++++ 4 files changed, 621 insertions(+), 17 deletions(-) create mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 51b8a07b4..e440caaf9 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -207,6 +207,12 @@ ${opentelemetry.version} test + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + io.opentelemetry opentelemetry-sdk-trace diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index ea85170fe..d491f9a93 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -34,6 +34,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import com.google.datastore.v1.CommitResponse; import com.google.datastore.v1.ExplainOptions; import com.google.datastore.v1.ReadOptions; import com.google.datastore.v1.ReserveIdsRequest; @@ -290,8 +291,6 @@ com.google.datastore.v1.RunQueryResponse runQuery( ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_RUN_QUERY : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_QUERY); com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); - span.setAttribute("isTransactional", isTransactional); - span.setAttribute("readConsistency", readOptions.getReadConsistency().toString()); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { RunQueryResponse response = @@ -303,10 +302,17 @@ com.google.datastore.v1.RunQueryResponse runQuery( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); span.addEvent( - spanName + ": Completed", + spanName, new ImmutableMap.Builder() - .put("Received", response.getBatch().getEntityResultsCount()) - .put("More results", response.getBatch().getMoreResults().toString()) + .put("response_count", response.getBatch().getEntityResultsCount()) + .put("transactional", isTransactional) + .put("read_consistency", readOptions.getReadConsistency().toString()) + .put( + "transaction_id", + (isTransactional + ? requestPb.getReadOptions().getTransaction().toStringUtf8() + : "")) + .put("more_results", response.getBatch().getMoreResults().toString()) .build()); return response; } catch (RetryHelperException e) { @@ -523,18 +529,18 @@ com.google.datastore.v1.LookupResponse lookup( ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_LOOKUP : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_LOOKUP); com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); - span.setAttribute("isTransactional", isTransactional); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( () -> { com.google.datastore.v1.LookupResponse response = datastoreRpc.lookup(requestPb); span.addEvent( - spanName + ": Completed", + spanName, new ImmutableMap.Builder() .put("Received", response.getFoundCount()) .put("Missing", response.getMissingCount()) .put("Deferred", response.getDeferredCount()) + .put("isTransactional", isTransactional) .build()); return response; }, @@ -690,15 +696,25 @@ com.google.datastore.v1.CommitResponse commit( ? com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_TRANSACTION_COMMIT : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_COMMIT; com.google.cloud.datastore.telemetry.TraceUtil.Span span = otelTraceUtil.startSpan(spanName); - span.setAttribute("isTransactional", isTransactional); try (com.google.cloud.datastore.telemetry.TraceUtil.Scope ignored = span.makeCurrent()) { - return RetryHelper.runWithRetries( - () -> datastoreRpc.commit(requestPb), - retrySettings, - requestPb.getTransaction().isEmpty() - ? EXCEPTION_HANDLER - : TRANSACTION_OPERATION_EXCEPTION_HANDLER, - getOptions().getClock()); + CommitResponse response = + RetryHelper.runWithRetries( + () -> datastoreRpc.commit(requestPb), + retrySettings, + requestPb.getTransaction().isEmpty() + ? EXCEPTION_HANDLER + : TRANSACTION_OPERATION_EXCEPTION_HANDLER, + getOptions().getClock()); + span.addEvent( + spanName, + new ImmutableMap.Builder() + .put("doc_count", response.getMutationResultsCount()) + .put("transactional", isTransactional) + .put( + "transaction_id", + isTransactional ? requestPb.getTransaction().toStringUtf8() : "") + .build()); + return response; } catch (RetryHelperException e) { span.end(e); throw DatastoreException.translateAndThrow(e); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index b8174184e..1de627158 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -334,7 +334,7 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY2 = - Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) + Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); KEY3 = @@ -342,7 +342,7 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY4 = - Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) + Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); // Set up the tracer for custom TraceID injection diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java new file mode 100644 index 000000000..6432cbc9e --- /dev/null +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java @@ -0,0 +1,582 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.it; + +import static com.google.cloud.datastore.telemetry.TraceUtil.*; +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.IncompleteKey; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.KeyFactory; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery.PropertyFilter; +import com.google.cloud.datastore.testing.RemoteDatastoreHelper; +import com.google.common.base.Preconditions; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.OpenTelemetrySdkBuilder; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.EventData; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public class ITTracingTest { + protected boolean isUsingGlobalOpenTelemetrySDK() { + return useGlobalOpenTelemetrySDK; + } + + protected String datastoreNamedDatabase() { + return datastoreNamedDatabase; + } + + private static final Logger logger = + Logger.getLogger(com.google.cloud.datastore.it.ITTracingTest.class.getName()); + + private static final int TRACE_FORCE_FLUSH_MILLIS = 1000; + private static final int TRACE_PROVIDER_SHUTDOWN_MILLIS = 1000; + private static final int IN_MEMORY_SPAN_EXPORTER_DELAY_MILLIS = 50; + private static final String SERVICE = "google.datastore.v1.Datastore/"; + + private static Key KEY1; + + private static Key KEY2; + + private static OpenTelemetrySdk openTelemetrySdk; + + // We use an InMemorySpanExporter for testing which keeps all generated trace spans + // in memory so that we can check their correctness. + protected InMemorySpanExporter inMemorySpanExporter; + private static DatastoreOptions options; + + protected Datastore datastore; + private static RemoteDatastoreHelper remoteDatastoreHelper; + + @TestParameter boolean useGlobalOpenTelemetrySDK; + + @TestParameter({ + /*(default)*/ + "", + "test-db" + }) + String datastoreNamedDatabase; + + Map spanNameToSpanId = new HashMap<>(); + Map spanIdToParentSpanId = new HashMap<>(); + Map spanNameToSpanData = new HashMap<>(); + + @Rule public TestName testName = new TestName(); + + @Before + public void before() { + inMemorySpanExporter = InMemorySpanExporter.create(); + + Resource resource = + Resource.getDefault().merge(Resource.builder().put(SERVICE_NAME, "Sparky").build()); + SpanProcessor inMemorySpanProcessor = SimpleSpanProcessor.create(inMemorySpanExporter); + DatastoreOptions.Builder optionsBuilder = DatastoreOptions.newBuilder(); + DatastoreOpenTelemetryOptions.Builder otelOptionsBuilder = + DatastoreOpenTelemetryOptions.newBuilder(); + OpenTelemetrySdkBuilder openTelemetrySdkBuilder = + OpenTelemetrySdk.builder() + .setTracerProvider( + SdkTracerProvider.builder() + .setResource(resource) + .addSpanProcessor(inMemorySpanProcessor) + .setSampler(Sampler.alwaysOn()) + .build()); + + if (isUsingGlobalOpenTelemetrySDK()) { + GlobalOpenTelemetry.resetForTest(); + openTelemetrySdk = openTelemetrySdkBuilder.buildAndRegisterGlobal(); + optionsBuilder.setOpenTelemetryOptions(otelOptionsBuilder.setTracingEnabled(true).build()); + } else { + openTelemetrySdk = openTelemetrySdkBuilder.build(); + optionsBuilder.setOpenTelemetryOptions( + otelOptionsBuilder.setTracingEnabled(true).setOpenTelemetry(openTelemetrySdk).build()); + } + + String namedDb = datastoreNamedDatabase(); + logger.log(Level.INFO, "Integration test using named database " + namedDb); + remoteDatastoreHelper = RemoteDatastoreHelper.create(namedDb, openTelemetrySdk); + options = remoteDatastoreHelper.getOptions(); + datastore = options.getService(); + + Preconditions.checkNotNull( + datastore, + "Error instantiating Datastore. Check that the service account credentials " + + "were properly set."); + + String projectId = options.getProjectId(); + String kind1 = "kind1"; + KEY1 = + Key.newBuilder(projectId, kind1, "key1", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY2 = + Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + + cleanupTestSpanContext(); + } + + @After + public void after() throws Exception { + if (isUsingGlobalOpenTelemetrySDK()) { + GlobalOpenTelemetry.resetForTest(); + } + remoteDatastoreHelper.deleteNamespace(); + inMemorySpanExporter.reset(); + CompletableResultCode completableResultCode = + openTelemetrySdk.getSdkTracerProvider().shutdown(); + completableResultCode.join(TRACE_PROVIDER_SHUTDOWN_MILLIS, TimeUnit.MILLISECONDS); + openTelemetrySdk = null; + } + + @AfterClass + public static void teardown() {} + + void waitForTracesToComplete() throws Exception { + // The same way that querying the Cloud Trace backend may not give us the + // full trace on the first try, querying the in-memory traces may not result + // in the full trace immediately. Note that performing the `flush` is not + // enough. This doesn't pose an issue in practice, but can make tests flaky. + // Therefore, we're adding a delay to make sure we avoid any flakiness. + inMemorySpanExporter.flush().join(IN_MEMORY_SPAN_EXPORTER_DELAY_MILLIS, TimeUnit.MILLISECONDS); + TimeUnit.MILLISECONDS.sleep(IN_MEMORY_SPAN_EXPORTER_DELAY_MILLIS); + + CompletableResultCode completableResultCode = + openTelemetrySdk.getSdkTracerProvider().forceFlush(); + completableResultCode.join(TRACE_FORCE_FLUSH_MILLIS, TimeUnit.MILLISECONDS); + } + + // Prepares all the spans in memory for inspection. + List prepareSpans() throws Exception { + waitForTracesToComplete(); + List spans = inMemorySpanExporter.getFinishedSpanItems(); + buildSpanMaps(spans); + printSpans(); + return spans; + } + + void buildSpanMaps(List spans) { + for (SpanData spanData : spans) { + spanNameToSpanData.put(spanData.getName(), spanData); + spanNameToSpanId.put(spanData.getName(), spanData.getSpanId()); + spanIdToParentSpanId.put(spanData.getSpanId(), spanData.getParentSpanId()); + } + } + + // Returns the SpanData object for the span with the given name. + // Returns null if no span with the given name exists. + @Nullable + SpanData getSpanByName(String spanName) { + return spanNameToSpanData.get(spanName); + } + + // Returns the SpanData object for the gRPC span with the given RPC name. + // Returns null if no such span exists. + @Nullable + SpanData getGrpcSpanByName(String rpcName) { + return getSpanByName(SERVICE + rpcName); + } + + String grpcSpanName(String rpcName) { + return SERVICE + rpcName; + } + + void assertSameTrace(SpanData... spans) { + if (spans.length > 1) { + String traceId = spans[0].getTraceId(); + for (SpanData spanData : spans) { + assertEquals(traceId, spanData.getTraceId()); + } + } + } + + // Helper to see the spans in standard output while developing tests + void printSpans() { + for (SpanData spanData : spanNameToSpanData.values()) { + logger.log( + Level.FINE, + String.format( + "SPAN ID:%s, ParentID:%s, KIND:%s, TRACE ID:%s, NAME:%s, ATTRIBUTES:%s, EVENTS:%s\n", + spanData.getSpanId(), + spanData.getParentSpanId(), + spanData.getKind(), + spanData.getTraceId(), + spanData.getName(), + spanData.getAttributes().toString(), + spanData.getEvents().toString())); + } + } + + // Asserts that the span hierarchy exists for the given span names. The hierarchy starts with the + // root span, followed + // by the child span, grandchild span, and so on. It also asserts that all the given spans belong + // to the same trace, + // and that datastore-generated spans contain the expected datastore attributes. + void assertSpanHierarchy(String... spanNamesHierarchy) { + List spanNames = Arrays.asList(spanNamesHierarchy); + + for (int i = 0; i + 1 < spanNames.size(); ++i) { + String parentSpanName = spanNames.get(i); + String childSpanName = spanNames.get(i + 1); + SpanData parentSpan = getSpanByName(parentSpanName); + SpanData childSpan = getSpanByName(childSpanName); + assertNotNull(parentSpan); + assertNotNull(childSpan); + assertEquals(childSpan.getParentSpanId(), parentSpan.getSpanId()); + assertSameTrace(childSpan, parentSpan); + // gRPC spans do not have datastore attributes. + if (!parentSpanName.startsWith(SERVICE)) { + assertHasExpectedAttributes(parentSpan); + } + if (!childSpanName.startsWith(SERVICE)) { + assertHasExpectedAttributes(childSpan); + } + } + } + + void assertHasExpectedAttributes(SpanData spanData, String... additionalExpectedAttributes) { + // All datastore-generated spans have the settings attributes. + List expectedAttributes = + Arrays.asList( + "gcp.datastore.memoryUtilization", + "gcp.datastore.settings.host", + "gcp.datastore.settings.databaseId", + "gcp.datastore.settings.channel.needsCredentials", + "gcp.datastore.settings.channel.needsEndpoint", + "gcp.datastore.settings.channel.needsHeaders", + "gcp.datastore.settings.channel.shouldAutoClose", + "gcp.datastore.settings.channel.transportName", + "gcp.datastore.settings.retrySettings.maxRpcTimeout", + "gcp.datastore.settings.retrySettings.retryDelayMultiplier", + "gcp.datastore.settings.retrySettings.initialRetryDelay", + "gcp.datastore.settings.credentials.authenticationType", + "gcp.datastore.settings.retrySettings.maxAttempts", + "gcp.datastore.settings.retrySettings.maxRetryDelay", + "gcp.datastore.settings.retrySettings.rpcTimeoutMultiplier", + "gcp.datastore.settings.retrySettings.totalTimeout", + "gcp.datastore.settings.retrySettings.initialRpcTimeout"); + + expectedAttributes.addAll(Arrays.asList(additionalExpectedAttributes)); + + Attributes spanAttributes = spanData.getAttributes(); + for (String expectedAttribute : expectedAttributes) { + assertNotNull(spanAttributes.get(AttributeKey.stringKey(expectedAttribute))); + } + } + + // Returns true if and only if the given span data contains an event with the given name and the + // given expected + // attributes. + boolean hasEvent(SpanData spanData, String eventName, @Nullable Attributes expectedAttributes) { + if (spanData == null) { + return false; + } + + logger.log( + Level.INFO, + String.format( + "Checking if span named '%s' (ID='%s') contains an event named '%s'", + spanData.getName(), spanData.getSpanId(), eventName)); + + List events = spanData.getEvents(); + for (EventData event : events) { + if (event.getName().equals(eventName)) { + if (expectedAttributes == null) { + return true; + } + + // Make sure attributes also match. + Attributes eventAttributes = event.getAttributes(); + return expectedAttributes.equals(eventAttributes); + } + } + return false; + } + + void cleanupTestSpanContext() { + inMemorySpanExporter.reset(); + spanNameToSpanId.clear(); + spanIdToParentSpanId.clear(); + spanNameToSpanData.clear(); + } + + // This is a POJO used for testing APIs that take a POJO. + public static class Pojo { + public int bar; + + public Pojo() { + bar = 0; + } + + public Pojo(int bar) { + this.bar = bar; + } + + public int getBar() { + return bar; + } + + public void setBar(int bar) { + this.bar = bar; + } + } + + @Test + public void lookupTraceTest() throws Exception { + Entity entity = datastore.get(KEY1); + assertNull(entity); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_LOOKUP); + SpanData span = getSpanByName(SPAN_NAME_LOOKUP); + assertTrue( + hasEvent( + span, + SPAN_NAME_LOOKUP, + Attributes.builder() + .put("Received", 0) + .put("Missing", 1) + .put("Deferred", 0) + .put("transactional", false) + .build())); + } + + @Test + public void allocateIdsTraceTest() throws Exception { + String kind1 = "kind1"; + KeyFactory keyFactory = datastore.newKeyFactory().setKind(kind1); + IncompleteKey pk1 = keyFactory.newKey(); + Key key1 = datastore.allocateId(pk1); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_ALLOCATE_IDS); + } + + @Test + public void reserveIdsTraceTest() throws Exception { + KeyFactory keyFactory = datastore.newKeyFactory().setKind("MyKind"); + Key key1 = keyFactory.newKey(10); + Key key2 = keyFactory.newKey("name"); + List keyList = datastore.reserveIds(key1, key2); + assertEquals(2, keyList.size()); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_RESERVE_IDS); + } + + @Test + public void commitTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + Entity response = datastore.add(entity1); + assertEquals(entity1, response); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_COMMIT); + } + + @Test + public void putTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + Entity response = datastore.put(entity1); + assertEquals(entity1, response); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_COMMIT); + } + + @Test + public void updateTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_COMMIT); + + SpanData spanData = getSpanByName(SPAN_NAME_COMMIT); + assertTrue( + hasEvent( + spanData, + SPAN_NAME_COMMIT, + Attributes.builder() + .put("doc_count", response.size()) + .put("transactional", false) + .put("transaction_id", "") + .build())); + + // Clean Up test span context to verify update spans + cleanupTestSpanContext(); + + Entity entity1_update = Entity.newBuilder(entity1).set("test_field", "new_test_value1").build(); + Entity entity2_update = Entity.newBuilder(entity2).set("test_field", "new_test_value1").build(); + datastore.update(entity1_update, entity2_update); + + spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_COMMIT); + } + + @Test + public void deleteTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_key", "test_value").build(); + Entity response = datastore.put(entity1); + assertEquals(entity1, response); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_COMMIT); + + SpanData spanData = getSpanByName(SPAN_NAME_COMMIT); + assertTrue( + hasEvent( + spanData, + SPAN_NAME_COMMIT, + Attributes.builder() + .put("doc_count", 1) + .put("transactional", false) + .put("transaction_id", "") + .build())); + + // Clean Up test span context to verify update spans + cleanupTestSpanContext(); + + datastore.delete(entity1.getKey()); + spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_COMMIT); + + spanData = getSpanByName(SPAN_NAME_COMMIT); + assertTrue( + hasEvent( + spanData, + SPAN_NAME_COMMIT, + Attributes.builder() + .put("doc_count", 1) + .put("transactional", false) + .put("transaction_id", "") + .build())); + } + + @Test + public void runQueryTraceTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + // Clean Up test span context to verify RunQuery spans + cleanupTestSpanContext(); + + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_RUN_QUERY); + + SpanData span = getSpanByName(SPAN_NAME_RUN_QUERY); + assertTrue( + hasEvent( + span, + SPAN_NAME_RUN_QUERY, + Attributes.builder() + .put("response_count", 1) + .put("transactional", false) + .put("read_consistency", "READ_CONSISTENCY_UNSPECIFIED") + .put("more_results", "NO_MORE_RESULTS") + .put("transaction_id", "") + .build())); + } + + @Test + public void runAggregationQueryTraceTest() throws Exception {} + + @Test + public void newTransactionReadTraceTest() throws Exception {} + + @Test + public void newTransactionQueryTest() throws Exception {} + + @Test + public void newTransactionReadWriteTraceTest() throws Exception {} + + @Test + public void newTransactionRollbackTest() throws Exception {} + + @Test + public void runInTransactionQueryTest() throws Exception {} +} From 6944e1b48b853c5f1b7a970a7d717b8d84ef0009 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:11:27 -0700 Subject: [PATCH 27/41] test: Additional Transaction tests and AggregationQuery test (#1518) * test: ReadWrite Transaction test * test: Added test for Transactional RunQuery and Transaction Rollback * test: runInTransaction API tracing test - Fixed setting of common span attributes to spans in runInTransaction - Removed some gRPC related channel attributes that are not present in this Datastore version, yet. --- .../google/cloud/datastore/DatastoreImpl.java | 12 +- .../telemetry/DisabledTraceUtil.java | 5 + .../datastore/telemetry/EnabledTraceUtil.java | 3 +- .../cloud/datastore/telemetry/TraceUtil.java | 7 + .../cloud/datastore/it/ITTracingTest.java | 292 +++++++++++++++++- 5 files changed, 303 insertions(+), 16 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index d491f9a93..3e9081d66 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -157,7 +157,7 @@ private io.opentelemetry.api.trace.Span startSpanWithParentContext( .with( io.opentelemetry.api.trace.Span.wrap( parentSpanContext.getSpanContext()))); - return spanBuilder.startSpan(); + return otelTraceUtil.addSettingsAttributesToCurrentSpan(spanBuilder).startSpan(); } @Override @@ -540,7 +540,10 @@ com.google.datastore.v1.LookupResponse lookup( .put("Received", response.getFoundCount()) .put("Missing", response.getMissingCount()) .put("Deferred", response.getDeferredCount()) - .put("isTransactional", isTransactional) + .put("transactional", isTransactional) + .put( + "transaction_id", + isTransactional ? readOptions.getTransaction().toStringUtf8() : "") .build()); return response; }, @@ -772,6 +775,11 @@ public Void call() throws DatastoreException { retrySettings, EXCEPTION_HANDLER, getOptions().getClock()); + span.addEvent( + com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_ROLLBACK, + new ImmutableMap.Builder() + .put("transaction_id", requestPb.getTransaction().toStringUtf8()) + .build()); } catch (RetryHelperException e) { span.end(e); throw DatastoreException.translateAndThrow(e); diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java index 6ba0fd81c..06941c721 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DisabledTraceUtil.java @@ -22,6 +22,7 @@ import com.google.cloud.datastore.telemetry.TraceUtil.SpanContext; import io.grpc.ManagedChannelBuilder; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerProvider; import java.util.Map; @@ -115,6 +116,10 @@ public TraceUtil.Span startSpan(String spanName, TraceUtil.SpanContext parentSpa return new Span(); } + public SpanBuilder addSettingsAttributesToCurrentSpan(SpanBuilder spanBuilder) { + return getTracer().spanBuilder("TRACING_DISABLED_NO_OP"); + } + @Nonnull @Override public TraceUtil.Span getCurrentSpan() { diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java index 438395cb1..3b962754d 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/EnabledTraceUtil.java @@ -238,7 +238,8 @@ public Scope makeCurrent() { } /** Applies the current Datastore instance settings as attributes to the current Span */ - private SpanBuilder addSettingsAttributesToCurrentSpan(SpanBuilder spanBuilder) { + @Override + public SpanBuilder addSettingsAttributesToCurrentSpan(SpanBuilder spanBuilder) { spanBuilder = spanBuilder.setAllAttributes( Attributes.builder() diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java index dd1dcf29e..dce53e952 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TraceUtil.java @@ -22,6 +22,7 @@ import com.google.cloud.datastore.DatastoreOptions; import io.grpc.ManagedChannelBuilder; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.Tracer; import java.util.Map; import javax.annotation.Nonnull; @@ -144,6 +145,12 @@ interface Scope extends AutoCloseable { */ Span startSpan(String spanName, SpanContext parentSpanContext); + /** + * Adds common SpanAttributes to the current span, useful when hand-creating a new Span without + * using the TraceUtil.Span interface. + */ + SpanBuilder addSettingsAttributesToCurrentSpan(SpanBuilder spanBuilder); + /** Returns the current span. */ @Nonnull Span getCurrentSpan(); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java index 6432cbc9e..485f3272e 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java @@ -16,7 +16,9 @@ package com.google.cloud.datastore.it; +import static com.google.cloud.datastore.aggregation.Aggregation.count; import static com.google.cloud.datastore.telemetry.TraceUtil.*; +import static com.google.common.truth.Truth.assertThat; import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -24,6 +26,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import com.google.cloud.datastore.AggregationQuery; +import com.google.cloud.datastore.AggregationResult; +import com.google.cloud.datastore.AggregationResults; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; import com.google.cloud.datastore.DatastoreOptions; @@ -33,7 +38,10 @@ import com.google.cloud.datastore.KeyFactory; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.ReadOption; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; +import com.google.cloud.datastore.Transaction; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.common.base.Preconditions; import com.google.testing.junit.testparameterinjector.TestParameter; @@ -91,6 +99,10 @@ protected String datastoreNamedDatabase() { private static Key KEY2; + private static Key KEY3; + + private static Key KEY4; + private static OpenTelemetrySdk openTelemetrySdk; // We use an InMemorySpanExporter for testing which keeps all generated trace spans @@ -166,7 +178,14 @@ public void before() { Key.newBuilder(projectId, kind1, "key2", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); - + KEY3 = + Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); + KEY4 = + Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) + .setNamespace(options.getNamespace()) + .build(); cleanupTestSpanContext(); } @@ -295,11 +314,6 @@ void assertHasExpectedAttributes(SpanData spanData, String... additionalExpected "gcp.datastore.memoryUtilization", "gcp.datastore.settings.host", "gcp.datastore.settings.databaseId", - "gcp.datastore.settings.channel.needsCredentials", - "gcp.datastore.settings.channel.needsEndpoint", - "gcp.datastore.settings.channel.needsHeaders", - "gcp.datastore.settings.channel.shouldAutoClose", - "gcp.datastore.settings.channel.transportName", "gcp.datastore.settings.retrySettings.maxRpcTimeout", "gcp.datastore.settings.retrySettings.retryDelayMultiplier", "gcp.datastore.settings.retrySettings.initialRetryDelay", @@ -380,6 +394,8 @@ public void lookupTraceTest() throws Exception { Entity entity = datastore.get(KEY1); assertNull(entity); + waitForTracesToComplete(); + List spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_LOOKUP); @@ -403,6 +419,8 @@ public void allocateIdsTraceTest() throws Exception { IncompleteKey pk1 = keyFactory.newKey(); Key key1 = datastore.allocateId(pk1); + waitForTracesToComplete(); + List spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_ALLOCATE_IDS); @@ -416,6 +434,8 @@ public void reserveIdsTraceTest() throws Exception { List keyList = datastore.reserveIds(key1, key2); assertEquals(2, keyList.size()); + waitForTracesToComplete(); + List spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_RESERVE_IDS); @@ -427,6 +447,8 @@ public void commitTraceTest() throws Exception { Entity response = datastore.add(entity1); assertEquals(entity1, response); + waitForTracesToComplete(); + List spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_COMMIT); @@ -438,6 +460,8 @@ public void putTraceTest() throws Exception { Entity response = datastore.put(entity1); assertEquals(entity1, response); + waitForTracesToComplete(); + List spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_COMMIT); @@ -476,6 +500,8 @@ public void updateTraceTest() throws Exception { Entity entity2_update = Entity.newBuilder(entity2).set("test_field", "new_test_value1").build(); datastore.update(entity1_update, entity2_update); + waitForTracesToComplete(); + spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_COMMIT); @@ -506,6 +532,9 @@ public void deleteTraceTest() throws Exception { cleanupTestSpanContext(); datastore.delete(entity1.getKey()); + + waitForTracesToComplete(); + spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_COMMIT); @@ -544,6 +573,8 @@ public void runQueryTraceTest() throws Exception { assertEquals(entity1, queryResults.next()); assertFalse(queryResults.hasNext()); + waitForTracesToComplete(); + List spans = prepareSpans(); assertEquals(1, spans.size()); assertSpanHierarchy(SPAN_NAME_RUN_QUERY); @@ -563,20 +594,255 @@ public void runQueryTraceTest() throws Exception { } @Test - public void runAggregationQueryTraceTest() throws Exception {} + public void runAggregationQueryTraceTest() throws Exception { + Entity entity1 = + Entity.newBuilder(KEY1) + .set("pepper_name", "jalapeno") + .set("max_scoville_level", 10000) + .build(); + Entity entity2 = + Entity.newBuilder(KEY2) + .set("pepper_name", "serrano") + .set("max_scoville_level", 25000) + .build(); + Entity entity3 = + Entity.newBuilder(KEY3) + .set("pepper_name", "habanero") + .set("max_scoville_level", 350000) + .build(); + Entity entity4 = + Entity.newBuilder(KEY4) + .set("pepper_name", "ghost") + .set("max_scoville_level", 1500000) + .build(); - @Test - public void newTransactionReadTraceTest() throws Exception {} + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + entityList.add(entity3); + entityList.add(entity4); + + List response = datastore.add(entity1, entity2, entity3, entity4); + assertEquals(entityList, response); + + // Clean Up test span context to verify RunAggregationQuery spans + cleanupTestSpanContext(); + + PropertyFilter mediumSpicyFilters = PropertyFilter.lt("max_scoville_level", 100000); + StructuredQuery mediumSpicyQuery = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(mediumSpicyFilters).build(); + AggregationQuery countSpicyPeppers = + Query.newAggregationQueryBuilder() + .addAggregation(count().as("count")) + .over(mediumSpicyQuery) + .build(); + AggregationResults results = datastore.runAggregation(countSpicyPeppers); + assertThat(results.size()).isEqualTo(1); + AggregationResult result = results.get(0); + assertThat(result.getLong("count")).isEqualTo(2L); + + waitForTracesToComplete(); + + List spans = prepareSpans(); + assertEquals(1, spans.size()); + assertSpanHierarchy(SPAN_NAME_RUN_AGGREGATION_QUERY); + } @Test - public void newTransactionQueryTest() throws Exception {} + public void newTransactionReadWriteTraceTest() throws Exception { + // Transaction.Begin + Transaction transaction = datastore.newTransaction(); + + // Transaction.Lookup + Entity entity = datastore.get(KEY1, ReadOption.transactionId(transaction.getTransactionId())); + assertNull(entity); + + Entity updatedEntity = Entity.newBuilder(KEY1).set("test_field", "new_test_value1").build(); + transaction.put(updatedEntity); + + // Transaction.Commit + transaction.commit(); + + waitForTracesToComplete(); + + List spans = prepareSpans(); + assertEquals(3, spans.size()); + + assertSpanHierarchy(SPAN_NAME_BEGIN_TRANSACTION); + assertSpanHierarchy(SPAN_NAME_TRANSACTION_LOOKUP); + SpanData span = getSpanByName(SPAN_NAME_TRANSACTION_LOOKUP); + assertTrue( + hasEvent( + span, + SPAN_NAME_TRANSACTION_LOOKUP, + Attributes.builder() + .put("Deferred", 0) + .put("Missing", 1) + .put("Received", 0) + .put("transactional", true) + .put("transaction_id", transaction.getTransactionId().toStringUtf8()) + .build())); + + assertSpanHierarchy(SPAN_NAME_TRANSACTION_COMMIT); + span = getSpanByName(SPAN_NAME_TRANSACTION_COMMIT); + assertTrue( + hasEvent( + span, + SPAN_NAME_TRANSACTION_COMMIT, + Attributes.builder() + .put("doc_count", 1) + .put("transactional", true) + .put("transaction_id", transaction.getTransactionId().toStringUtf8()) + .build())); + } @Test - public void newTransactionReadWriteTraceTest() throws Exception {} + public void newTransactionQueryTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + // Clean Up test span context to verify Transaction RunQuery spans + cleanupTestSpanContext(); + + Transaction transaction = datastore.newTransaction(); + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + QueryResults queryResults = transaction.run(query); + transaction.commit(); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + + waitForTracesToComplete(); + + List spans = prepareSpans(); + assertEquals(3, spans.size()); + + assertSpanHierarchy(SPAN_NAME_BEGIN_TRANSACTION); + assertSpanHierarchy(SPAN_NAME_TRANSACTION_RUN_QUERY); + assertSpanHierarchy(SPAN_NAME_TRANSACTION_COMMIT); + SpanData span = getSpanByName(SPAN_NAME_TRANSACTION_RUN_QUERY); + assertTrue( + hasEvent( + span, + SPAN_NAME_TRANSACTION_RUN_QUERY, + Attributes.builder() + .put("response_count", 1) + .put("transactional", true) + .put("read_consistency", "READ_CONSISTENCY_UNSPECIFIED") + .put("more_results", "NO_MORE_RESULTS") + .put("transaction_id", transaction.getTransactionId().toStringUtf8()) + .build())); + } @Test - public void newTransactionRollbackTest() throws Exception {} + public void newTransactionRollbackTest() throws Exception { + Entity entity1 = Entity.newBuilder(KEY1).set("pepper_type", "jalapeno").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("pepper_type", "habanero").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + // Clean Up test span context to verify Transaction Rollback spans + cleanupTestSpanContext(); + + String simplified_spice_level = "not_spicy"; + Entity entity1update = + Entity.newBuilder(entity1).set("spice_level", simplified_spice_level).build(); + Transaction transaction = datastore.newTransaction(); + entity1 = transaction.get(KEY1); + switch (entity1.getString("pepper_type")) { + case "jalapeno": + simplified_spice_level = "mild"; + break; + + case "habanero": + simplified_spice_level = "hot"; + break; + } + transaction.update(entity1update); + transaction.delete(KEY2); + transaction.rollback(); + assertFalse(transaction.isActive()); + + waitForTracesToComplete(); + + List spans = prepareSpans(); + assertEquals(3, spans.size()); + + assertSpanHierarchy(SPAN_NAME_BEGIN_TRANSACTION); + assertSpanHierarchy(SPAN_NAME_TRANSACTION_LOOKUP); + SpanData span = getSpanByName(SPAN_NAME_TRANSACTION_LOOKUP); + assertTrue( + hasEvent( + span, + SPAN_NAME_TRANSACTION_LOOKUP, + Attributes.builder() + .put("Deferred", 0) + .put("Missing", 0) + .put("Received", 1) + .put("transactional", true) + .put("transaction_id", transaction.getTransactionId().toStringUtf8()) + .build())); + + assertSpanHierarchy(SPAN_NAME_ROLLBACK); + span = getSpanByName(SPAN_NAME_ROLLBACK); + assertTrue( + hasEvent( + span, + SPAN_NAME_ROLLBACK, + Attributes.builder() + .put("transaction_id", transaction.getTransactionId().toStringUtf8()) + .build())); + } @Test - public void runInTransactionQueryTest() throws Exception {} + public void runInTransactionQueryTest() throws Exception { + // Set up + Entity entity1 = Entity.newBuilder(KEY1).set("test_field", "test_value1").build(); + Entity entity2 = Entity.newBuilder(KEY2).set("test_field", "test_value2").build(); + List entityList = new ArrayList<>(); + entityList.add(entity1); + entityList.add(entity2); + + List response = datastore.add(entity1, entity2); + assertEquals(entityList, response); + + // Clean Up test span context to verify Transaction Rollback spans + cleanupTestSpanContext(); + + PropertyFilter filter = PropertyFilter.eq("test_field", entity1.getValue("test_field")); + Query query = + Query.newEntityQueryBuilder().setKind(KEY1.getKind()).setFilter(filter).build(); + Datastore.TransactionCallable callable = + transaction -> { + QueryResults queryResults = datastore.run(query); + assertTrue(queryResults.hasNext()); + assertEquals(entity1, queryResults.next()); + assertFalse(queryResults.hasNext()); + return true; + }; + datastore.runInTransaction(callable); + + waitForTracesToComplete(); + + List spans = prepareSpans(); + assertEquals(4, spans.size()); + + // Since the runInTransaction method runs the TransactionCallable opaquely in a transaction + // there is no way for the API user to know the transaction ID, so we will not validate it here. + assertSpanHierarchy(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_BEGIN_TRANSACTION); + assertSpanHierarchy(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_RUN_QUERY); + assertSpanHierarchy(SPAN_NAME_TRANSACTION_RUN, SPAN_NAME_TRANSACTION_COMMIT); + } } From 5652dd452fcaa382db8a30ecdf53e287fa93ce66 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:19:43 -0700 Subject: [PATCH 28/41] fix: Undelete gRPC upgrade docs --- README.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/README.md b/README.md index 4520f2779..308b4669b 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,102 @@ running on Compute Engine or from your own desktop. To run the example on App En the code from the main method to your application's servlet class and change the print statements to display on your webpage. +gRPC Java Datastore Client User Guide +------- +In this feature launch, the [Java Datastore client](https://github.com/googleapis/java-datastore) now offers gRPC as a transport layer option with experimental support. Using [gRPC connection pooling](https://grpc.io/docs/guides/performance/) enables distributing RPCs over multiple connections which may improve performance. + +#### Download Instructions +Instructions: +1. Clone the grpc-experimental branch from GitHub: +```python +git clone -b grpc-experimental https://github.com/googleapis/java-datastore.git +``` +2. Run the following commands to build the library: +```python +# Go to the directory the code was downloaded to +cd java-datastore/ + +# Build the library +mvn clean install -DskipTests=true +``` +3. Add the following dependency to your project: +```xml + + com.google.cloud + google-cloud-datastore + 2.20.0-grpc-experimental-1-SNAPSHOT + +``` + +#### How to Use +To opt-in to the gRPC transport behavior, simply add the below line of code (`setTransportOptions`) to your Datastore client instantiation. + +Example: +```java +DatastoreOptions datastoreOptions = + DatastoreOptions.newBuilder() + .setProjectId("my-project") + .setDatabaseId("my-database") + .setTransportOptions(GrpcTransportOptions.newBuilder().build()) + .build(); +``` +Setting the transport options explicitly to `GrpcTransportOptions` will signal the client to use gRPC instead of HTTP when making calls to the server. + +To revert back to the existing stable behavior and transport, simply remove the transport options line or replace it with `HttpTransportOptions`. Please note this will require an application rebuild and restart. +Example: +```java +// will default to existing HTTP transport behavior +DatastoreOptions datastoreOptions = DatastoreOptions.newBuilder() + .setProjectId("my-project") + .setDatabaseId("my-database") + .build(); + +// will also default to existing HTTP transport behavior +DatastoreOptions datastoreOptions = + DatastoreOptions.newBuilder() + .setProjectId("my-project") + .setDatabaseId("my-database") + .setTransportOptions(HttpTransportOptions.newBuilder() + .setConnectTimeout(1000) + .build()).build(); +``` + +Note: client instantiations that already use `setTransportOptions` with `HttpTransportOptions` will continue to have the same behavior. Only transports that are explicitly set to gRPC will change. + +#### Verify Datastore Transport Options Type +To verify which type of TransportOptions you have successfully configured, you can use the below lines of code to compare transport options type: +```java +// checks if using gRPC transport options +boolean isGRPC = datastore.getOptions().getTransportOptions() instanceof GrpcTransportOptions; + +// checks if using HTTP transport options +boolean isHTTP = datastore.getOptions().getTransportOptions() instanceof HTTPTransportOptions; +``` + +#### New Features +There are new gRPC specific features available to use in this update. + +##### Channel Pooling +To customize the number of channels your client uses, you can update the channel provider in the DatastoreOptions. +See [ChannelPoolSettings](https://cloud.google.com/java/docs/reference/gax/latest/com.google.api.gax.grpc.ChannelPoolSettings) and [Performance Best Practices](https://grpc.io/docs/guides/performance/) for more information on channel pools and best practices for performance. + +Example: +```java +InstantiatingGrpcChannelProvider channelProvider = + DatastoreSettings.defaultGrpcTransportProviderBuilder() + .setChannelPoolSettings( + ChannelPoolSettings.builder() + .setInitialChannelCount(MIN_VAL) + .setMaxChannelCount(MAX_VAL) + .build()) + .build(); + +DatastoreOptions options = DatastoreOptions.newBuilder() + .setProjectId("my-project") + .setChannelProvider(channelProvider) + .setTransportOptions(GrpcTransportOptions.newBuilder().build()) + .build(); +``` Testing ------- From aa00fe206f67896b9c70a1179a65d762584f7dc5 Mon Sep 17 00:00:00 2001 From: jimit-j-shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:23:34 -0700 Subject: [PATCH 29/41] fix: Undo merge mistakes --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 308b4669b..9089655ec 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.40.0 + 26.41.0 pom import @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.40.0') +implementation platform('com.google.cloud:libraries-bom:26.42.0') implementation 'com.google.cloud:google-cloud-datastore' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-datastore:2.20.0' +implementation 'com.google.cloud:google-cloud-datastore:2.20.2' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.20.0" +libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.20.2" ``` @@ -286,7 +286,7 @@ boolean isHTTP = datastore.getOptions().getTransportOptions() instanceof HTTPTra There are new gRPC specific features available to use in this update. ##### Channel Pooling -To customize the number of channels your client uses, you can update the channel provider in the DatastoreOptions. +To customize the number of channels your client uses, you can update the channel provider in the DatastoreOptions. See [ChannelPoolSettings](https://cloud.google.com/java/docs/reference/gax/latest/com.google.api.gax.grpc.ChannelPoolSettings) and [Performance Best Practices](https://grpc.io/docs/guides/performance/) for more information on channel pools and best practices for performance. Example: @@ -480,7 +480,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-datastore/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-datastore.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.20.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.20.2 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From 3a3d7b8a32a14a839eb811e96533907395836768 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Mon, 5 Aug 2024 23:29:37 -0700 Subject: [PATCH 30/41] fix: Updating span event strings (#1539) * fix: Fixing user-facing span names in line with go/firestore-client-trace-catalog * fix: updating bom dependency version to fix https://github.com/googleapis/java-datastore/actions/runs/10256441634/job/28375496112?pr=1539 --- google-cloud-datastore/pom.xml | 2 +- .../google/cloud/datastore/DatastoreImpl.java | 8 +++---- .../cloud/datastore/it/ITTracingTest.java | 23 ++++++++++--------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index e440caaf9..d58746cd9 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -16,7 +16,7 @@ google-cloud-datastore - 1.38.0 + 1.39.0 diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index 3e9081d66..e4db9620b 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -302,9 +302,9 @@ com.google.datastore.v1.RunQueryResponse runQuery( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); span.addEvent( - spanName, + spanName + " complete.", new ImmutableMap.Builder() - .put("response_count", response.getBatch().getEntityResultsCount()) + .put("doc_count", response.getBatch().getEntityResultsCount()) .put("transactional", isTransactional) .put("read_consistency", readOptions.getReadConsistency().toString()) .put( @@ -535,7 +535,7 @@ com.google.datastore.v1.LookupResponse lookup( () -> { com.google.datastore.v1.LookupResponse response = datastoreRpc.lookup(requestPb); span.addEvent( - spanName, + spanName + " complete.", new ImmutableMap.Builder() .put("Received", response.getFoundCount()) .put("Missing", response.getMissingCount()) @@ -709,7 +709,7 @@ com.google.datastore.v1.CommitResponse commit( : TRANSACTION_OPERATION_EXCEPTION_HANDLER, getOptions().getClock()); span.addEvent( - spanName, + spanName + " complete.", new ImmutableMap.Builder() .put("doc_count", response.getMutationResultsCount()) .put("transactional", isTransactional) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java index 485f3272e..85ff4758b 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITTracingTest.java @@ -403,12 +403,13 @@ public void lookupTraceTest() throws Exception { assertTrue( hasEvent( span, - SPAN_NAME_LOOKUP, + SPAN_NAME_LOOKUP + " complete.", Attributes.builder() .put("Received", 0) .put("Missing", 1) .put("Deferred", 0) .put("transactional", false) + .put("transaction_id", "") .build())); } @@ -486,7 +487,7 @@ public void updateTraceTest() throws Exception { assertTrue( hasEvent( spanData, - SPAN_NAME_COMMIT, + SPAN_NAME_COMMIT + " complete.", Attributes.builder() .put("doc_count", response.size()) .put("transactional", false) @@ -521,7 +522,7 @@ public void deleteTraceTest() throws Exception { assertTrue( hasEvent( spanData, - SPAN_NAME_COMMIT, + SPAN_NAME_COMMIT + " complete.", Attributes.builder() .put("doc_count", 1) .put("transactional", false) @@ -543,7 +544,7 @@ public void deleteTraceTest() throws Exception { assertTrue( hasEvent( spanData, - SPAN_NAME_COMMIT, + SPAN_NAME_COMMIT + " complete.", Attributes.builder() .put("doc_count", 1) .put("transactional", false) @@ -583,9 +584,9 @@ public void runQueryTraceTest() throws Exception { assertTrue( hasEvent( span, - SPAN_NAME_RUN_QUERY, + SPAN_NAME_RUN_QUERY + " complete.", Attributes.builder() - .put("response_count", 1) + .put("doc_count", 1) .put("transactional", false) .put("read_consistency", "READ_CONSISTENCY_UNSPECIFIED") .put("more_results", "NO_MORE_RESULTS") @@ -674,7 +675,7 @@ public void newTransactionReadWriteTraceTest() throws Exception { assertTrue( hasEvent( span, - SPAN_NAME_TRANSACTION_LOOKUP, + SPAN_NAME_TRANSACTION_LOOKUP + " complete.", Attributes.builder() .put("Deferred", 0) .put("Missing", 1) @@ -688,7 +689,7 @@ public void newTransactionReadWriteTraceTest() throws Exception { assertTrue( hasEvent( span, - SPAN_NAME_TRANSACTION_COMMIT, + SPAN_NAME_TRANSACTION_COMMIT + " complete.", Attributes.builder() .put("doc_count", 1) .put("transactional", true) @@ -732,9 +733,9 @@ public void newTransactionQueryTest() throws Exception { assertTrue( hasEvent( span, - SPAN_NAME_TRANSACTION_RUN_QUERY, + SPAN_NAME_TRANSACTION_RUN_QUERY + " complete.", Attributes.builder() - .put("response_count", 1) + .put("doc_count", 1) .put("transactional", true) .put("read_consistency", "READ_CONSISTENCY_UNSPECIFIED") .put("more_results", "NO_MORE_RESULTS") @@ -786,7 +787,7 @@ public void newTransactionRollbackTest() throws Exception { assertTrue( hasEvent( span, - SPAN_NAME_TRANSACTION_LOOKUP, + SPAN_NAME_TRANSACTION_LOOKUP + " complete.", Attributes.builder() .put("Deferred", 0) .put("Missing", 0) From 3640051cf45a350125758bbf6cc4f4daa1d47363 Mon Sep 17 00:00:00 2001 From: Jimit Shah <57637300+jimit-j-shah@users.noreply.github.com> Date: Wed, 4 Sep 2024 10:07:09 -0700 Subject: [PATCH 31/41] Fix: typo in test causing integration test failure (#1556) https://btx.cloud.google.com/invocations/c11a2e8b-4494-4ddc-a77e-cf2bcbcf5254/targets/cloud-devrel%2Fclient-libraries%2Fjava%2Fjava-datastore%2Fpresubmit%2Fintegration;config=default/log --- .../java/com/google/cloud/datastore/it/ITE2ETracingTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java index 1de627158..bee54a1f0 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITE2ETracingTest.java @@ -338,7 +338,7 @@ public void before() throws Exception { .setNamespace(options.getNamespace()) .build(); KEY3 = - Key.newBuilder(projectId, kind1, "key4", options.getDatabaseId()) + Key.newBuilder(projectId, kind1, "key3", options.getDatabaseId()) .setNamespace(options.getNamespace()) .build(); KEY4 = @@ -381,9 +381,6 @@ public void after() throws Exception { @AfterClass public static void teardown() throws Exception { traceClient_v1.close(); - CompletableResultCode completableResultCode = - openTelemetrySdk.getSdkTracerProvider().shutdown(); - completableResultCode.join(TRACE_PROVIDER_SHUTDOWN_MILLIS, TimeUnit.MILLISECONDS); } // Generates a random hex string of length `numBytes` From f787871e9190c6f6023dd1de06d9466c4766790b Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 5 Aug 2024 21:24:20 +0000 Subject: [PATCH 32/41] chore(main): release 2.21.0 (#1517) :robot: I have created a release *beep* *boop* --- ## [2.21.0](https://togithub.com/googleapis/java-datastore/compare/v2.20.2...v2.21.0) (2024-07-31) ### Features * Enable hermetic library generation ([#1462](https://togithub.com/googleapis/java-datastore/issues/1462)) ([d142d9c](https://togithub.com/googleapis/java-datastore/commit/d142d9c95d91c8cadaf696efc12d6136814938ff)) --- This PR was generated with [Release Please](https://togithub.com/googleapis/release-please). See [documentation](https://togithub.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 10 files changed, 35 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ca1c263..52e107131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [2.21.0](https://github.com/googleapis/java-datastore/compare/v2.20.2...v2.21.0) (2024-07-31) + + +### Features + +* Enable hermetic library generation ([#1462](https://github.com/googleapis/java-datastore/issues/1462)) ([d142d9c](https://github.com/googleapis/java-datastore/commit/d142d9c95d91c8cadaf696efc12d6136814938ff)) + ## [2.20.2](https://github.com/googleapis/java-datastore/compare/v2.20.1...v2.20.2) (2024-06-28) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 70b9590b0..59a18afe9 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.20.3-SNAPSHOT + 2.21.0 com.google.cloud google-cloud-datastore-parent - 2.20.3-SNAPSHOT + 2.21.0 jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 281c581c9..87cc26a83 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.20.3-SNAPSHOT + 2.21.0 pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.20.3-SNAPSHOT + 2.21.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.20.3-SNAPSHOT + 2.21.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.111.3-SNAPSHOT + 0.112.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.20.3-SNAPSHOT + 2.21.0 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index d58746cd9..9fdf0e285 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.20.3-SNAPSHOT + 2.21.0 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.20.3-SNAPSHOT + 2.21.0 google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index 69da06680..c92a2e085 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.20.3-SNAPSHOT + 2.21.0 grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.20.3-SNAPSHOT + 2.21.0 diff --git a/pom.xml b/pom.xml index 407041f2b..92e9e6b9c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.20.3-SNAPSHOT + 2.21.0 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.20.3-SNAPSHOT + 2.21.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.20.3-SNAPSHOT + 2.21.0 com.google.cloud google-cloud-datastore - 2.20.3-SNAPSHOT + 2.21.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.111.3-SNAPSHOT + 0.112.0 com.google.cloud.datastore datastore-v1-proto-client - 2.20.3-SNAPSHOT + 2.21.0 com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index f53996a6d..712d421f2 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.20.3-SNAPSHOT + 2.21.0 proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.20.3-SNAPSHOT + 2.21.0 diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index 85f530c75..6a4c4c026 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.111.3-SNAPSHOT + 0.112.0 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.20.3-SNAPSHOT + 2.21.0 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 93710c03b..18c99a152 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.20.3-SNAPSHOT + 2.21.0 diff --git a/versions.txt b/versions.txt index 5d2ffbced..5dca750af 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.20.2:2.20.3-SNAPSHOT -google-cloud-datastore-bom:2.20.2:2.20.3-SNAPSHOT -proto-google-cloud-datastore-v1:0.111.2:0.111.3-SNAPSHOT -datastore-v1-proto-client:2.20.2:2.20.3-SNAPSHOT -proto-google-cloud-datastore-admin-v1:2.20.2:2.20.3-SNAPSHOT -grpc-google-cloud-datastore-admin-v1:2.20.2:2.20.3-SNAPSHOT +google-cloud-datastore:2.21.0:2.21.0 +google-cloud-datastore-bom:2.21.0:2.21.0 +proto-google-cloud-datastore-v1:0.112.0:0.112.0 +datastore-v1-proto-client:2.21.0:2.21.0 +proto-google-cloud-datastore-admin-v1:2.21.0:2.21.0 +grpc-google-cloud-datastore-admin-v1:2.21.0:2.21.0 From a8805485dff565a67ede08df14cb5f25f122f72e Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 6 Aug 2024 16:35:45 +0200 Subject: [PATCH 33/41] deps: update dependency com.google.cloud:sdk-platform-java-config to v3.33.0 (#1531) --- .github/workflows/unmanaged_dependency_check.yaml | 2 +- .kokoro/presubmit/graalvm-native-17.cfg | 2 +- .kokoro/presubmit/graalvm-native.cfg | 2 +- google-cloud-datastore-bom/pom.xml | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index ec72b08ac..ffc858543 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -14,6 +14,6 @@ jobs: shell: bash run: .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.32.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.33.0 with: bom-path: google-cloud-datastore-bom/pom.xml diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index 0c813eaf7..a452e7c8c 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.32.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.33.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index ef1c9176c..ad15d4f7f 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.32.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.33.0" } env_vars: { diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 87cc26a83..f1be69a62 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud sdk-platform-java-config - 3.32.0 + 3.33.0 Google Cloud datastore BOM diff --git a/pom.xml b/pom.xml index 92e9e6b9c..5f7573e98 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.32.0 + 3.33.0 From fb43b236693a3a3cf03b66f9d307db8d77d0f35d Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:02:31 -0700 Subject: [PATCH 34/41] chore(main): release 2.21.1-SNAPSHOT (#1538) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 59a18afe9..58425b4b1 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.21.0 + 2.21.1-SNAPSHOT com.google.cloud google-cloud-datastore-parent - 2.21.0 + 2.21.1-SNAPSHOT jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index f1be69a62..fa6abfeda 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.21.0 + 2.21.1-SNAPSHOT pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.21.0 + 2.21.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.0 + 2.21.1-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.0 + 0.112.1-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.0 + 2.21.1-SNAPSHOT diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 9fdf0e285..836d14a7a 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.21.0 + 2.21.1-SNAPSHOT jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.21.0 + 2.21.1-SNAPSHOT google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index c92a2e085..c74367bf9 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.0 + 2.21.1-SNAPSHOT grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.0 + 2.21.1-SNAPSHOT diff --git a/pom.xml b/pom.xml index 5f7573e98..85eeafc8e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.21.0 + 2.21.1-SNAPSHOT Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.0 + 2.21.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.0 + 2.21.1-SNAPSHOT com.google.cloud google-cloud-datastore - 2.21.0 + 2.21.1-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.0 + 0.112.1-SNAPSHOT com.google.cloud.datastore datastore-v1-proto-client - 2.21.0 + 2.21.1-SNAPSHOT com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index 712d421f2..533ad8ad3 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.0 + 2.21.1-SNAPSHOT proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.0 + 2.21.1-SNAPSHOT diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index 6a4c4c026..b8bb530fa 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.0 + 0.112.1-SNAPSHOT proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.21.0 + 2.21.1-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 18c99a152..b182ec319 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.21.0 + 2.21.1-SNAPSHOT diff --git a/versions.txt b/versions.txt index 5dca750af..5141114d4 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.21.0:2.21.0 -google-cloud-datastore-bom:2.21.0:2.21.0 -proto-google-cloud-datastore-v1:0.112.0:0.112.0 -datastore-v1-proto-client:2.21.0:2.21.0 -proto-google-cloud-datastore-admin-v1:2.21.0:2.21.0 -grpc-google-cloud-datastore-admin-v1:2.21.0:2.21.0 +google-cloud-datastore:2.21.0:2.21.1-SNAPSHOT +google-cloud-datastore-bom:2.21.0:2.21.1-SNAPSHOT +proto-google-cloud-datastore-v1:0.112.0:0.112.1-SNAPSHOT +datastore-v1-proto-client:2.21.0:2.21.1-SNAPSHOT +proto-google-cloud-datastore-admin-v1:2.21.0:2.21.1-SNAPSHOT +grpc-google-cloud-datastore-admin-v1:2.21.0:2.21.1-SNAPSHOT From 2fdc90a253d0107b37fe85e797763d37027e344d Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Tue, 6 Aug 2024 20:15:59 +0200 Subject: [PATCH 35/41] chore(deps): update dependency com.google.cloud:libraries-bom to v26.43.0 (#1515) --- samples/native-image-sample/pom.xml | 2 +- samples/snippets/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 0f3c1919c..5a0ee2386 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -28,7 +28,7 @@ com.google.cloud libraries-bom - 26.41.0 + 26.43.0 pom import diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index f04cdc885..7e4de0be6 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 26.41.0 + 26.43.0 pom import From 1706dd58bed12200c89a3a63d896fdbb00f61b3c Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:45:05 -0700 Subject: [PATCH 36/41] chore(main): release 2.21.1 (#1540) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 10 files changed, 35 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52e107131..f6234e13b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [2.21.1](https://github.com/googleapis/java-datastore/compare/v2.21.0...v2.21.1) (2024-08-06) + + +### Dependencies + +* Update dependency com.google.cloud:sdk-platform-java-config to v3.33.0 ([#1531](https://github.com/googleapis/java-datastore/issues/1531)) ([9e52395](https://github.com/googleapis/java-datastore/commit/9e52395f7ee71315331790284d35e7aad2f387ed)) + ## [2.21.0](https://github.com/googleapis/java-datastore/compare/v2.20.2...v2.21.0) (2024-07-31) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 58425b4b1..eb7a28713 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.21.1-SNAPSHOT + 2.21.1 com.google.cloud google-cloud-datastore-parent - 2.21.1-SNAPSHOT + 2.21.1 jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index fa6abfeda..c683a9474 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.21.1-SNAPSHOT + 2.21.1 pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.21.1-SNAPSHOT + 2.21.1 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.1-SNAPSHOT + 2.21.1 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.1-SNAPSHOT + 0.112.1 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.1-SNAPSHOT + 2.21.1 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 836d14a7a..d2a214bb9 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.21.1-SNAPSHOT + 2.21.1 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.21.1-SNAPSHOT + 2.21.1 google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index c74367bf9..373333ce3 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.1-SNAPSHOT + 2.21.1 grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.1-SNAPSHOT + 2.21.1 diff --git a/pom.xml b/pom.xml index 85eeafc8e..72d70b386 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.21.1-SNAPSHOT + 2.21.1 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.1-SNAPSHOT + 2.21.1 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.1-SNAPSHOT + 2.21.1 com.google.cloud google-cloud-datastore - 2.21.1-SNAPSHOT + 2.21.1 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.1-SNAPSHOT + 0.112.1 com.google.cloud.datastore datastore-v1-proto-client - 2.21.1-SNAPSHOT + 2.21.1 com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index 533ad8ad3..1c660d581 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.1-SNAPSHOT + 2.21.1 proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.1-SNAPSHOT + 2.21.1 diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index b8bb530fa..4740a89c4 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.1-SNAPSHOT + 0.112.1 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.21.1-SNAPSHOT + 2.21.1 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index b182ec319..3d3b267eb 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.21.1-SNAPSHOT + 2.21.1 diff --git a/versions.txt b/versions.txt index 5141114d4..478c0b62e 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.21.0:2.21.1-SNAPSHOT -google-cloud-datastore-bom:2.21.0:2.21.1-SNAPSHOT -proto-google-cloud-datastore-v1:0.112.0:0.112.1-SNAPSHOT -datastore-v1-proto-client:2.21.0:2.21.1-SNAPSHOT -proto-google-cloud-datastore-admin-v1:2.21.0:2.21.1-SNAPSHOT -grpc-google-cloud-datastore-admin-v1:2.21.0:2.21.1-SNAPSHOT +google-cloud-datastore:2.21.1:2.21.1 +google-cloud-datastore-bom:2.21.1:2.21.1 +proto-google-cloud-datastore-v1:0.112.1:0.112.1 +datastore-v1-proto-client:2.21.1:2.21.1 +proto-google-cloud-datastore-admin-v1:2.21.1:2.21.1 +grpc-google-cloud-datastore-admin-v1:2.21.1:2.21.1 From 0b496ba8855fff7b65a83b225e5536bb8a6f2ae0 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:28:27 +0000 Subject: [PATCH 37/41] chore(main): release 2.21.2-SNAPSHOT (#1541) :robot: I have created a release *beep* *boop* --- ### Updating meta-information for bleeding-edge SNAPSHOT release. --- This PR was generated with [Release Please](https://togithub.com/googleapis/release-please). See [documentation](https://togithub.com/googleapis/release-please#release-please). --- datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index eb7a28713..38ede6074 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.21.1 + 2.21.2-SNAPSHOT com.google.cloud google-cloud-datastore-parent - 2.21.1 + 2.21.2-SNAPSHOT jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index c683a9474..234946cf0 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.21.1 + 2.21.2-SNAPSHOT pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.21.1 + 2.21.2-SNAPSHOT com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.1 + 2.21.2-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.1 + 0.112.2-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.1 + 2.21.2-SNAPSHOT diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index d2a214bb9..dd6b6b149 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.21.1 + 2.21.2-SNAPSHOT jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.21.1 + 2.21.2-SNAPSHOT google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index 373333ce3..793e30422 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.1 + 2.21.2-SNAPSHOT grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.1 + 2.21.2-SNAPSHOT diff --git a/pom.xml b/pom.xml index 72d70b386..4fcdfc0e2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.21.1 + 2.21.2-SNAPSHOT Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.1 + 2.21.2-SNAPSHOT com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.1 + 2.21.2-SNAPSHOT com.google.cloud google-cloud-datastore - 2.21.1 + 2.21.2-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.1 + 0.112.2-SNAPSHOT com.google.cloud.datastore datastore-v1-proto-client - 2.21.1 + 2.21.2-SNAPSHOT com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index 1c660d581..4032680c9 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.1 + 2.21.2-SNAPSHOT proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.1 + 2.21.2-SNAPSHOT diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index 4740a89c4..0e9cb5736 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.1 + 0.112.2-SNAPSHOT proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.21.1 + 2.21.2-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 3d3b267eb..aea2bd2f3 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.21.1 + 2.21.2-SNAPSHOT diff --git a/versions.txt b/versions.txt index 478c0b62e..8b0985906 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.21.1:2.21.1 -google-cloud-datastore-bom:2.21.1:2.21.1 -proto-google-cloud-datastore-v1:0.112.1:0.112.1 -datastore-v1-proto-client:2.21.1:2.21.1 -proto-google-cloud-datastore-admin-v1:2.21.1:2.21.1 -grpc-google-cloud-datastore-admin-v1:2.21.1:2.21.1 +google-cloud-datastore:2.21.1:2.21.2-SNAPSHOT +google-cloud-datastore-bom:2.21.1:2.21.2-SNAPSHOT +proto-google-cloud-datastore-v1:0.112.1:0.112.2-SNAPSHOT +datastore-v1-proto-client:2.21.1:2.21.2-SNAPSHOT +proto-google-cloud-datastore-admin-v1:2.21.1:2.21.2-SNAPSHOT +grpc-google-cloud-datastore-admin-v1:2.21.1:2.21.2-SNAPSHOT From 16b351521c95a30d7f92b2dc61d8751c89a6e1af Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Tue, 20 Aug 2024 13:59:45 -0400 Subject: [PATCH 38/41] chore: secure hermetic_library_generation workflow (#1552) * chore: secure hermetic_library_generation workflow Thanks to @diogoteles08 for the inspection on our repos. This PR inlines environment variables to avoid overriding script injections. * fix wording --- .github/workflows/hermetic_library_generation.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 7146cc3dc..ab23b9fec 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -17,10 +17,14 @@ name: Hermetic library generation upon generation config change through pull req on: pull_request: +env: + HEAD_REF: ${{ github.head_ref }} + REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }} + jobs: library_generation: # skip pull requests coming from a forked repository - if: github.event.pull_request.head.repo.full_name == github.repository + if: github.env.REPO_FULL_NAME == github.repository runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -30,11 +34,11 @@ jobs: - name: Generate changed libraries shell: bash run: | - set -x + set -ex [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash .github/scripts/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ - --current_branch ${{ github.head_ref }} + --current_branch $HEAD_REF env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From ea98972539d40d492b0901522edc525a8fd62492 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Thu, 22 Aug 2024 19:30:09 +0200 Subject: [PATCH 39/41] deps: update dependency com.google.cloud:sdk-platform-java-config to v3.34.0 (#1547) --- .github/workflows/unmanaged_dependency_check.yaml | 2 +- .kokoro/presubmit/graalvm-native-17.cfg | 2 +- .kokoro/presubmit/graalvm-native.cfg | 2 +- google-cloud-datastore-bom/pom.xml | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index ffc858543..84c86037f 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -14,6 +14,6 @@ jobs: shell: bash run: .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.33.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.34.0 with: bom-path: google-cloud-datastore-bom/pom.xml diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index a452e7c8c..639683e72 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.33.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.34.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index ad15d4f7f..b2121a633 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.33.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.34.0" } env_vars: { diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 234946cf0..435b6e894 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud sdk-platform-java-config - 3.33.0 + 3.34.0 Google Cloud datastore BOM diff --git a/pom.xml b/pom.xml index 4fcdfc0e2..85ab3c70b 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.33.0 + 3.34.0 From fb4bd08c834b4aabc10b42b50b23b8d46232800f Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:14:44 -0700 Subject: [PATCH 40/41] chore(main): release 2.21.2 (#1553) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 10 files changed, 35 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6234e13b..e953fa33f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [2.21.2](https://github.com/googleapis/java-datastore/compare/v2.21.1...v2.21.2) (2024-08-22) + + +### Dependencies + +* Update dependency com.google.cloud:sdk-platform-java-config to v3.34.0 ([#1547](https://github.com/googleapis/java-datastore/issues/1547)) ([8c5f595](https://github.com/googleapis/java-datastore/commit/8c5f5954d88732ab929b4477a3f15b0052adc2ff)) + ## [2.21.1](https://github.com/googleapis/java-datastore/compare/v2.21.0...v2.21.1) (2024-08-06) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 38ede6074..ae1f83917 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.21.2-SNAPSHOT + 2.21.2 com.google.cloud google-cloud-datastore-parent - 2.21.2-SNAPSHOT + 2.21.2 jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 435b6e894..63beb11f1 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.21.2-SNAPSHOT + 2.21.2 pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.21.2-SNAPSHOT + 2.21.2 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.2-SNAPSHOT + 2.21.2 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.2-SNAPSHOT + 0.112.2 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.2-SNAPSHOT + 2.21.2 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index dd6b6b149..d5e18642c 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.21.2-SNAPSHOT + 2.21.2 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.21.2-SNAPSHOT + 2.21.2 google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index 793e30422..6527e96f3 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.2-SNAPSHOT + 2.21.2 grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.2-SNAPSHOT + 2.21.2 diff --git a/pom.xml b/pom.xml index 85ab3c70b..fda01b132 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.21.2-SNAPSHOT + 2.21.2 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.2-SNAPSHOT + 2.21.2 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.21.2-SNAPSHOT + 2.21.2 com.google.cloud google-cloud-datastore - 2.21.2-SNAPSHOT + 2.21.2 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.2-SNAPSHOT + 0.112.2 com.google.cloud.datastore datastore-v1-proto-client - 2.21.2-SNAPSHOT + 2.21.2 com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index 4032680c9..a8c91f4dd 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.21.2-SNAPSHOT + 2.21.2 proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.21.2-SNAPSHOT + 2.21.2 diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index 0e9cb5736..bf896574d 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.112.2-SNAPSHOT + 0.112.2 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.21.2-SNAPSHOT + 2.21.2 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index aea2bd2f3..5f2e900a8 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.21.2-SNAPSHOT + 2.21.2 diff --git a/versions.txt b/versions.txt index 8b0985906..3cee21923 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.21.1:2.21.2-SNAPSHOT -google-cloud-datastore-bom:2.21.1:2.21.2-SNAPSHOT -proto-google-cloud-datastore-v1:0.112.1:0.112.2-SNAPSHOT -datastore-v1-proto-client:2.21.1:2.21.2-SNAPSHOT -proto-google-cloud-datastore-admin-v1:2.21.1:2.21.2-SNAPSHOT -grpc-google-cloud-datastore-admin-v1:2.21.1:2.21.2-SNAPSHOT +google-cloud-datastore:2.21.2:2.21.2 +google-cloud-datastore-bom:2.21.2:2.21.2 +proto-google-cloud-datastore-v1:0.112.2:0.112.2 +datastore-v1-proto-client:2.21.2:2.21.2 +proto-google-cloud-datastore-admin-v1:2.21.2:2.21.2 +grpc-google-cloud-datastore-admin-v1:2.21.2:2.21.2 From 3be4fb7b4923f18f4d75c14676421e92f3f08d24 Mon Sep 17 00:00:00 2001 From: Mridula <66699525+mpeddada1@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:16:23 -0400 Subject: [PATCH 41/41] chore: remove datastore native image sample in favor of sample hosted in google-cloud-java (#1519) --- samples/native-image-sample/README.md | 96 ------------ samples/native-image-sample/pom.xml | 141 ------------------ .../datastore/NativeImageDatastoreSample.java | 131 ---------------- .../ITNativeImageDatastoreSample.java | 75 ---------- samples/pom.xml | 1 - 5 files changed, 444 deletions(-) delete mode 100644 samples/native-image-sample/README.md delete mode 100644 samples/native-image-sample/src/main/java/com/example/datastore/NativeImageDatastoreSample.java delete mode 100644 samples/native-image-sample/src/test/java/com/example/datastore/ITNativeImageDatastoreSample.java diff --git a/samples/native-image-sample/README.md b/samples/native-image-sample/README.md deleted file mode 100644 index 5f2cfbd27..000000000 --- a/samples/native-image-sample/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Datastore Sample Application with Native Image - -This application uses the [Google Cloud Datastore client library](https://cloud.google.com/datastore/docs/reference/libraries) and is compatible with Native Image compilation. - -This sample runs through some basic operations of creating/deleting entities, running queries, and running transaction code. - -## Setup Instructions - -You will need to follow these prerequisite steps in order to run the samples: - -1. If you have not already, [create a Google Cloud Platform Project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project). - -2. Install the [Google Cloud SDK](https://cloud.google.com/sdk/) which will allow you to run the sample with your project's credentials. - - Once installed, log in with Application Default Credentials using the following command: - - ``` - gcloud auth application-default login - ``` - - **Note:** Authenticating with Application Default Credentials is convenient to use during development, but we recommend [alternate methods of authentication](https://cloud.google.com/docs/authentication/production) during production use. - -3. Install the native image compiler. - - You can follow the [installation instructions](https://www.graalvm.org/docs/getting-started/#install-graalvm). - After following the instructions, ensure that you install the native image extension installed by running: - - ``` - gu install native-image - ``` - - Once you finish following the instructions, verify that the default version of Java is set to the correct version by running `java -version` in a terminal. - - You will see something similar to the below output: - - ``` - $ java -version - - openjdk version "17.0.3" 2022-04-19 - OpenJDK Runtime Environment GraalVM CE 22.1.0 (build 17.0.3+7-jvmci-22.1-b06) - OpenJDK 64-Bit Server VM GraalVM CE 22.1.0 (build 17.0.3+7-jvmci-22.1-b06, mixed mode, sharing) - ``` -## Sample -1. **(Optional)** If you wish to run the application against the [Datastore emulator](https://cloud.google.com/sdk/gcloud/reference/beta/emulators/datastore), ensure that you have the [Google Cloud SDK](https://cloud.google.com/sdk) installed. - - In a new terminal window, start the emulator via `gcloud`: - - ``` - gcloud beta emulators datastore start --host-port=localhost:9010 - ``` - - Leave the emulator running in this terminal for now. - In the next section, we will run the sample application against the Datastore emulator instance. - -2. Navigate to this directory and compile the application with the native image compiler. - - ``` - mvn package -P native -DskipTests - ``` - -3. **(Optional)** If you're using the emulator, export the `DATASTORE_EMULATOR_HOST` as an environment variable in your terminal. - - ``` - export DATASTORE_EMULATOR_HOST=localhost:9010 - ``` - - The Datastore Client Libraries will detect this environment variable and automatically connect to the emulator instance if this variable is set. - -4. Run the application. - - ``` - ./target/native-image-sample - ``` - -5. The application will run through some basic Datastore operations and log some output statements. - - ``` - Successfully added entity. - Reading entity: 1cf34cc1-2b8a-4945-9fc4-058f03dcd08e - Successfully deleted entity: 1cf34cc1-2b8a-4945-9fc4-058f03dcd08e - Run fake transaction code. - Found entity: - name=de4f36f4-3936-4252-98d3-e0d56d485254 - kind=test-kind - namespace=nativeimage-test-namespace - properties={description=StringValue{valueType=STRING, excludeFromIndexes=false, meaning=0, value=hello world}} - Ran transaction callable. - ``` - -### Sample Integration test with Native Image Support - -In order to run the sample integration test as a native image, call the following command: - - ``` - mvn test -Pnative - ``` diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 5a0ee2386..e69de29bb 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -1,141 +0,0 @@ - - - 4.0.0 - com.example.datastore - native-image-sample - Native Image Sample - https://github.com/googleapis/java-datastore - - - - com.google.cloud.samples - shared-configuration - 1.2.0 - - - - - 1.8 - 1.8 - UTF-8 - - - - - - com.google.cloud - libraries-bom - 26.43.0 - pom - import - - - - - - - com.google.cloud - google-cloud-datastore - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.4.3 - test - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - com.example.datastore.NativeImageDatastoreSample - - - - - - - - - - - native - - - - org.junit.vintage - junit-vintage-engine - 5.10.2 - test - - - org.graalvm.buildtools - junit-platform-native - 0.10.2 - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - 3.2.5 - - - **/IT* - - - - - org.graalvm.buildtools - native-maven-plugin - 0.10.2 - true - - com.example.datastore.NativeImageDatastoreSample - - --no-fallback - --no-server - - - - - build-native - - build - test - - package - - - test-native - - test - - test - - - - - - - - \ No newline at end of file diff --git a/samples/native-image-sample/src/main/java/com/example/datastore/NativeImageDatastoreSample.java b/samples/native-image-sample/src/main/java/com/example/datastore/NativeImageDatastoreSample.java deleted file mode 100644 index 7ce5c900a..000000000 --- a/samples/native-image-sample/src/main/java/com/example/datastore/NativeImageDatastoreSample.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2020-2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.datastore; - -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.Key; -import com.google.cloud.datastore.Query; -import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.StructuredQuery; -import com.google.cloud.datastore.Transaction; -import java.time.Duration; -import java.time.Instant; -import java.util.UUID; - -/** Sample Datastore Application. */ -public class NativeImageDatastoreSample { - - /* Datastore namespace where entities will be created. */ - private static final String TEST_NAMESPACE = "nativeimage-test-namespace"; - - /* Datastore kind used. */ - private static final String TEST_KIND = "test-kind"; - - /** Entrypoint to the Datastore sample application. */ - public static void main(String[] args) { - Instant startTime = Instant.now(); - Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); - - String testId = UUID.randomUUID().toString(); - - addEntity(datastore, testId); - getEntity(datastore, testId); - deleteEntity(datastore, testId); - - runTransaction(datastore); - - String id = UUID.randomUUID().toString(); - Key key = createKey(datastore, id); - runTransactionCallable(datastore, key); - Instant endTime = Instant.now(); - Duration duration = Duration.between(startTime, endTime); - System.out.println("Duration: " + duration.toString()); - } - - static void addEntity(Datastore datastore, String id) { - Key key = createKey(datastore, id); - Entity entity = Entity.newBuilder(key).set("description", "hello world").build(); - datastore.add(entity); - System.out.println("Successfully added entity."); - } - - static void getEntity(Datastore datastore, String id) { - Key key = createKey(datastore, id); - Entity entity = datastore.get(key); - System.out.println("Reading entity: " + entity.getKey().getName()); - } - - static void deleteEntity(Datastore datastore, String id) { - Key key = createKey(datastore, id); - datastore.delete(key); - - Entity entity = datastore.get(key); - if (entity == null) { - System.out.println("Successfully deleted entity: " + id); - } else { - throw new RuntimeException("Failed to delete entity: " + id); - } - } - - static void runTransactionCallable(Datastore datastore, Key entityKey) { - datastore.runInTransaction( - client -> { - Entity entity = Entity.newBuilder(entityKey).set("description", "hello world").build(); - datastore.add(entity); - - StructuredQuery query = - Query.newEntityQueryBuilder().setNamespace(TEST_NAMESPACE).setKind(TEST_KIND).build(); - - QueryResults results = datastore.run(query); - while (results.hasNext()) { - Entity result = results.next(); - String name = result.getKey().getName(); - String kind = result.getKey().getKind(); - String namespace = result.getKey().getNamespace(); - System.out.println( - "Found entity:" - + "\n\t\tname=" - + name - + "\n\t\tkind=" - + kind - + "\n\t\tnamespace=" - + namespace - + "\n\t\tproperties=" - + result.getProperties().toString()); - } - - datastore.delete(entityKey); - return null; - }); - - System.out.println("Ran transaction callable."); - } - - private static void runTransaction(Datastore datastore) { - Transaction transaction = datastore.newTransaction(); - transaction.commit(); - transaction = datastore.newTransaction(); - transaction.rollback(); - System.out.println("Run fake transaction code."); - } - - static Key createKey(Datastore datastore, String id) { - return datastore.newKeyFactory().setNamespace(TEST_NAMESPACE).setKind(TEST_KIND).newKey(id); - } -} diff --git a/samples/native-image-sample/src/test/java/com/example/datastore/ITNativeImageDatastoreSample.java b/samples/native-image-sample/src/test/java/com/example/datastore/ITNativeImageDatastoreSample.java deleted file mode 100644 index 710f18367..000000000 --- a/samples/native-image-sample/src/test/java/com/example/datastore/ITNativeImageDatastoreSample.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.datastore; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.Key; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.UUID; -import org.junit.Before; -import org.junit.Test; - -/** Tests for {@link com.example.datastore.NativeImageDatastoreSample} */ -public class ITNativeImageDatastoreSample { - - private Datastore datastore; - private ByteArrayOutputStream bout; - private PrintStream out; - - @Before - public void setUp() { - datastore = DatastoreOptions.getDefaultInstance().getService(); - bout = new ByteArrayOutputStream(); - out = new PrintStream(bout); - System.setOut(out); - } - - @Test - public void testAddAndGetEntity() { - bout.reset(); - String testId = "test-id-" + UUID.randomUUID(); - NativeImageDatastoreSample.addEntity(datastore, testId); - NativeImageDatastoreSample.getEntity(datastore, testId); - assertThat(bout.toString()).contains("Reading entity: " + testId); - - NativeImageDatastoreSample.deleteEntity(datastore, testId); - } - - @Test - public void testRunTransactionalCallable() { - bout.reset(); - String testId = "test-id-" + UUID.randomUUID(); - Key key = NativeImageDatastoreSample.createKey(datastore, testId); - NativeImageDatastoreSample.runTransactionCallable(datastore, key); - assertThat(bout.toString()) - .contains( - "Found entity:" - + "\n\t\tname=" - + testId - + "\n\t\tkind=test-kind" - + "\n\t\tnamespace=nativeimage-test-namespace" - + "\n\t\tproperties={description=StringValue{valueType=STRING, excludeFromIndexes=false," - + " meaning=0, value=hello world}}\n" - + "Ran transaction callable."); - - NativeImageDatastoreSample.deleteEntity(datastore, "test-id"); - } -} diff --git a/samples/pom.xml b/samples/pom.xml index e81bec450..2e970d081 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -31,7 +31,6 @@ install-without-bom snapshot snippets - native-image-sample