diff --git a/README.md b/README.md index bac33a9b29..f9a31670e7 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.44.0') +implementation platform('com.google.cloud:libraries-bom:26.45.0') implementation 'com.google.cloud:google-cloud-bigtable' ``` diff --git a/google-cloud-bigtable/clirr-ignored-differences.xml b/google-cloud-bigtable/clirr-ignored-differences.xml index fb5b514b3d..8ddcb6fdf0 100644 --- a/google-cloud-bigtable/clirr-ignored-differences.xml +++ b/google-cloud-bigtable/clirr-ignored-differences.xml @@ -259,4 +259,10 @@ com/google/cloud/bigtable/admin/v2/models/Type$Int64$Encoding$BigEndianBytes * + + + 7004 + com/google/cloud/bigtable/admin/v2/stub/EnhancedBigtableTableAdminStub + * + diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java index f640bb6a30..889598020a 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java @@ -46,6 +46,7 @@ import com.google.cloud.bigtable.admin.v2.internal.NameUtil; import com.google.cloud.bigtable.admin.v2.models.AuthorizedView; import com.google.cloud.bigtable.admin.v2.models.Backup; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; import com.google.cloud.bigtable.admin.v2.models.CopyBackupRequest; import com.google.cloud.bigtable.admin.v2.models.CreateAuthorizedViewRequest; import com.google.cloud.bigtable.admin.v2.models.CreateBackupRequest; @@ -61,6 +62,7 @@ import com.google.cloud.bigtable.admin.v2.models.UpdateBackupRequest; import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest; import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub; +import com.google.cloud.bigtable.data.v2.internal.TableAdminRequestContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -154,8 +156,10 @@ public static BigtableTableAdminClient create( /** Constructs an instance of BigtableTableAdminClient with the given settings. */ public static BigtableTableAdminClient create(@Nonnull BigtableTableAdminSettings settings) throws IOException { + TableAdminRequestContext requestContext = + TableAdminRequestContext.create(settings.getProjectId(), settings.getInstanceId()); EnhancedBigtableTableAdminStub stub = - EnhancedBigtableTableAdminStub.createEnhanced(settings.getStubSettings()); + EnhancedBigtableTableAdminStub.createEnhanced(settings.getStubSettings(), requestContext); return create(settings.getProjectId(), settings.getInstanceId(), stub); } @@ -917,6 +921,11 @@ public void awaitReplication(String tableId) { stub.awaitReplicationCallable().futureCall(tableName)); } + public void awaitConsistency(ConsistencyRequest consistencyRequest) { + ApiExceptions.callAndTranslateApiException( + stub.awaitConsistencyCallable().futureCall(consistencyRequest)); + } + /** * Creates a backup with the specified configuration. * diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/ConsistencyRequest.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/ConsistencyRequest.java new file mode 100644 index 0000000000..0718af03c1 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/ConsistencyRequest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.admin.v2.models; + +import com.google.api.core.InternalApi; +import com.google.auto.value.AutoValue; +import com.google.bigtable.admin.v2.CheckConsistencyRequest; +import com.google.bigtable.admin.v2.DataBoostReadLocalWrites; +import com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest; +import com.google.bigtable.admin.v2.StandardReadRemoteWrites; +import com.google.bigtable.admin.v2.TableName; +import com.google.cloud.bigtable.data.v2.internal.TableAdminRequestContext; +import javax.annotation.Nonnull; + +@AutoValue +public abstract class ConsistencyRequest { + @Nonnull + protected abstract String getTableId(); + + @Nonnull + protected abstract CheckConsistencyRequest.ModeCase getMode(); + + public static ConsistencyRequest forReplication(String tableId) { + return new AutoValue_ConsistencyRequest( + tableId, CheckConsistencyRequest.ModeCase.STANDARD_READ_REMOTE_WRITES); + } + + public static ConsistencyRequest forDataBoost(String tableId) { + return new AutoValue_ConsistencyRequest( + tableId, CheckConsistencyRequest.ModeCase.DATA_BOOST_READ_LOCAL_WRITES); + } + + @InternalApi + public CheckConsistencyRequest toCheckConsistencyProto( + TableAdminRequestContext requestContext, String token) { + CheckConsistencyRequest.Builder builder = CheckConsistencyRequest.newBuilder(); + TableName tableName = + TableName.of(requestContext.getProjectId(), requestContext.getInstanceId(), getTableId()); + + if (getMode().equals(CheckConsistencyRequest.ModeCase.STANDARD_READ_REMOTE_WRITES)) { + builder.setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()); + } else { + builder.setDataBoostReadLocalWrites(DataBoostReadLocalWrites.newBuilder().build()); + } + + return builder.setName(tableName.toString()).setConsistencyToken(token).build(); + } + + @InternalApi + public GenerateConsistencyTokenRequest toGenerateTokenProto( + TableAdminRequestContext requestContext) { + GenerateConsistencyTokenRequest.Builder builder = GenerateConsistencyTokenRequest.newBuilder(); + TableName tableName = + TableName.of(requestContext.getProjectId(), requestContext.getInstanceId(), getTableId()); + + return builder.setName(tableName.toString()).build(); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitConsistencyCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitConsistencyCallable.java new file mode 100644 index 0000000000..7cdcb66599 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitConsistencyCallable.java @@ -0,0 +1,195 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.admin.v2.stub; + +import com.google.api.core.ApiAsyncFunction; +import com.google.api.core.ApiFunction; +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.gax.retrying.ExponentialPollAlgorithm; +import com.google.api.gax.retrying.NonCancellableFuture; +import com.google.api.gax.retrying.ResultRetryAlgorithm; +import com.google.api.gax.retrying.RetryAlgorithm; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.retrying.RetryingExecutor; +import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.retrying.ScheduledRetryingExecutor; +import com.google.api.gax.retrying.TimedAttemptSettings; +import com.google.api.gax.rpc.ApiCallContext; +import com.google.api.gax.rpc.ClientContext; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.bigtable.admin.v2.CheckConsistencyRequest; +import com.google.bigtable.admin.v2.CheckConsistencyResponse; +import com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest; +import com.google.bigtable.admin.v2.GenerateConsistencyTokenResponse; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; +import com.google.cloud.bigtable.data.v2.internal.TableAdminRequestContext; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.MoreExecutors; +import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; + +/** + * Callable that waits until either replication or Data Boost has caught up to the point it was + * called. + * + *

This callable wraps GenerateConsistencyToken and CheckConsistency RPCs. It will generate a + * token then poll until isConsistent is true. + */ +class AwaitConsistencyCallable extends UnaryCallable { + private final UnaryCallable + generateCallable; + private final UnaryCallable checkCallable; + private final RetryingExecutor executor; + + private final TableAdminRequestContext requestContext; + + static AwaitConsistencyCallable create( + UnaryCallable + generateCallable, + UnaryCallable checkCallable, + ClientContext clientContext, + RetrySettings pollingSettings, + TableAdminRequestContext requestContext) { + + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new PollResultAlgorithm(), + new ExponentialPollAlgorithm(pollingSettings, clientContext.getClock())); + + RetryingExecutor retryingExecutor = + new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); + + return new AwaitConsistencyCallable( + generateCallable, checkCallable, retryingExecutor, requestContext); + } + + @VisibleForTesting + AwaitConsistencyCallable( + UnaryCallable + generateCallable, + UnaryCallable checkCallable, + RetryingExecutor executor, + TableAdminRequestContext requestContext) { + this.generateCallable = generateCallable; + this.checkCallable = checkCallable; + this.executor = executor; + this.requestContext = requestContext; + } + + @Override + public ApiFuture futureCall( + final ConsistencyRequest consistencyRequest, final ApiCallContext apiCallContext) { + ApiFuture tokenFuture = + generateToken(consistencyRequest.toGenerateTokenProto(requestContext), apiCallContext); + + return ApiFutures.transformAsync( + tokenFuture, + new ApiAsyncFunction() { + @Override + public ApiFuture apply(GenerateConsistencyTokenResponse input) { + CheckConsistencyRequest request = + consistencyRequest.toCheckConsistencyProto( + requestContext, input.getConsistencyToken()); + return pollToken(request, apiCallContext); + } + }, + MoreExecutors.directExecutor()); + } + + private ApiFuture generateToken( + GenerateConsistencyTokenRequest generateRequest, ApiCallContext context) { + return generateCallable.futureCall(generateRequest, context); + } + + private ApiFuture pollToken(CheckConsistencyRequest request, ApiCallContext context) { + AttemptCallable attemptCallable = + new AttemptCallable<>(checkCallable, request, context); + RetryingFuture retryingFuture = + executor.createFuture(attemptCallable); + attemptCallable.setExternalFuture(retryingFuture); + attemptCallable.call(); + + return ApiFutures.transform( + retryingFuture, + new ApiFunction() { + @Override + public Void apply(CheckConsistencyResponse input) { + return null; + } + }, + MoreExecutors.directExecutor()); + } + + /** A callable representing an attempt to make an RPC call. */ + private static class AttemptCallable implements Callable { + private final UnaryCallable callable; + private final RequestT request; + + private volatile RetryingFuture externalFuture; + private volatile ApiCallContext callContext; + + AttemptCallable( + UnaryCallable callable, RequestT request, ApiCallContext callContext) { + this.callable = callable; + this.request = request; + this.callContext = callContext; + } + + void setExternalFuture(RetryingFuture externalFuture) { + this.externalFuture = externalFuture; + } + + @Override + public ResponseT call() { + try { + // NOTE: unlike gax's AttemptCallable, this ignores rpc timeouts + externalFuture.setAttemptFuture(new NonCancellableFuture()); + if (externalFuture.isDone()) { + return null; + } + ApiFuture internalFuture = callable.futureCall(request, callContext); + externalFuture.setAttemptFuture(internalFuture); + } catch (Throwable e) { + externalFuture.setAttemptFuture(ApiFutures.immediateFailedFuture(e)); + } + + return null; + } + } + + /** + * A polling algorithm for waiting for a consistent {@link CheckConsistencyResponse}. Please note + * that this class doesn't handle retryable errors and expects the underlying callable chain to + * handle this. + */ + private static class PollResultAlgorithm + implements ResultRetryAlgorithm { + @Override + public TimedAttemptSettings createNextAttempt( + Throwable prevThrowable, + CheckConsistencyResponse prevResponse, + TimedAttemptSettings prevSettings) { + return null; + } + + @Override + public boolean shouldRetry(Throwable prevThrowable, CheckConsistencyResponse prevResponse) + throws CancellationException { + return prevResponse != null && !prevResponse.getConsistent(); + } + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallable.java index a09026f7f7..2cb8549f5d 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallable.java @@ -15,31 +15,12 @@ */ package com.google.cloud.bigtable.admin.v2.stub; -import com.google.api.core.ApiAsyncFunction; -import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.api.gax.retrying.ExponentialPollAlgorithm; -import com.google.api.gax.retrying.NonCancellableFuture; -import com.google.api.gax.retrying.ResultRetryAlgorithm; -import com.google.api.gax.retrying.RetryAlgorithm; -import com.google.api.gax.retrying.RetrySettings; -import com.google.api.gax.retrying.RetryingExecutor; -import com.google.api.gax.retrying.RetryingFuture; -import com.google.api.gax.retrying.ScheduledRetryingExecutor; -import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.ApiCallContext; -import com.google.api.gax.rpc.ClientContext; import com.google.api.gax.rpc.UnaryCallable; -import com.google.bigtable.admin.v2.CheckConsistencyRequest; -import com.google.bigtable.admin.v2.CheckConsistencyResponse; -import com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest; -import com.google.bigtable.admin.v2.GenerateConsistencyTokenResponse; import com.google.bigtable.admin.v2.TableName; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.MoreExecutors; -import java.util.concurrent.Callable; -import java.util.concurrent.CancellationException; /** * Callable that waits until replication has caught up to the point it was called. @@ -47,144 +28,25 @@ *

This callable wraps GenerateConsistencyToken and CheckConsistency RPCs. It will generate a * token then poll until isConsistent is true. */ +/** @deprecated Please use {@link AwaitConsistencyCallable instead. */ +@Deprecated class AwaitReplicationCallable extends UnaryCallable { - private final UnaryCallable - generateCallable; - private final UnaryCallable checkCallable; - private final RetryingExecutor executor; + private final AwaitConsistencyCallable awaitConsistencyCallable; - static AwaitReplicationCallable create( - UnaryCallable - generateCallable, - UnaryCallable checkCallable, - ClientContext clientContext, - RetrySettings pollingSettings) { + static AwaitReplicationCallable create(AwaitConsistencyCallable awaitConsistencyCallable) { - RetryAlgorithm retryAlgorithm = - new RetryAlgorithm<>( - new PollResultAlgorithm(), - new ExponentialPollAlgorithm(pollingSettings, clientContext.getClock())); - - RetryingExecutor retryingExecutor = - new ScheduledRetryingExecutor<>(retryAlgorithm, clientContext.getExecutor()); - - return new AwaitReplicationCallable(generateCallable, checkCallable, retryingExecutor); - } - - @VisibleForTesting - AwaitReplicationCallable( - UnaryCallable - generateCallable, - UnaryCallable checkCallable, - RetryingExecutor executor) { - this.generateCallable = generateCallable; - this.checkCallable = checkCallable; - this.executor = executor; + return new AwaitReplicationCallable(awaitConsistencyCallable); } @Override public ApiFuture futureCall(final TableName tableName, final ApiCallContext context) { - ApiFuture tokenFuture = generateToken(tableName, context); - - return ApiFutures.transformAsync( - tokenFuture, - new ApiAsyncFunction() { - @Override - public ApiFuture apply(GenerateConsistencyTokenResponse input) { - CheckConsistencyRequest request = - CheckConsistencyRequest.newBuilder() - .setName(tableName.toString()) - .setConsistencyToken(input.getConsistencyToken()) - .build(); - - return pollToken(request, context); - } - }, - MoreExecutors.directExecutor()); - } - - private ApiFuture generateToken( - TableName tableName, ApiCallContext context) { - GenerateConsistencyTokenRequest generateRequest = - GenerateConsistencyTokenRequest.newBuilder().setName(tableName.toString()).build(); - return generateCallable.futureCall(generateRequest, context); - } + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(tableName.getTable()); - private ApiFuture pollToken(CheckConsistencyRequest request, ApiCallContext context) { - AttemptCallable attemptCallable = - new AttemptCallable<>(checkCallable, request, context); - RetryingFuture retryingFuture = - executor.createFuture(attemptCallable); - attemptCallable.setExternalFuture(retryingFuture); - attemptCallable.call(); - - return ApiFutures.transform( - retryingFuture, - new ApiFunction() { - @Override - public Void apply(CheckConsistencyResponse input) { - return null; - } - }, - MoreExecutors.directExecutor()); - } - - /** A callable representing an attempt to make an RPC call. */ - private static class AttemptCallable implements Callable { - private final UnaryCallable callable; - private final RequestT request; - - private volatile RetryingFuture externalFuture; - private volatile ApiCallContext callContext; - - AttemptCallable( - UnaryCallable callable, RequestT request, ApiCallContext callContext) { - this.callable = callable; - this.request = request; - this.callContext = callContext; - } - - void setExternalFuture(RetryingFuture externalFuture) { - this.externalFuture = externalFuture; - } - - @Override - public ResponseT call() { - try { - // NOTE: unlike gax's AttemptCallable, this ignores rpc timeouts - externalFuture.setAttemptFuture(new NonCancellableFuture()); - if (externalFuture.isDone()) { - return null; - } - ApiFuture internalFuture = callable.futureCall(request, callContext); - externalFuture.setAttemptFuture(internalFuture); - } catch (Throwable e) { - externalFuture.setAttemptFuture(ApiFutures.immediateFailedFuture(e)); - } - - return null; - } + return awaitConsistencyCallable.futureCall(consistencyRequest, context); } - /** - * A polling algorithm for waiting for a consistent {@link CheckConsistencyResponse}. Please note - * that this class doesn't handle retryable errors and expects the underlying callable chain to - * handle this. - */ - private static class PollResultAlgorithm - implements ResultRetryAlgorithm { - @Override - public TimedAttemptSettings createNextAttempt( - Throwable prevThrowable, - CheckConsistencyResponse prevResponse, - TimedAttemptSettings prevSettings) { - return null; - } - - @Override - public boolean shouldRetry(Throwable prevThrowable, CheckConsistencyResponse prevResponse) - throws CancellationException { - return prevResponse != null && !prevResponse.getConsistent(); - } + @VisibleForTesting + AwaitReplicationCallable(AwaitConsistencyCallable awaitConsistencyCallable) { + this.awaitConsistencyCallable = awaitConsistencyCallable; } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/EnhancedBigtableTableAdminStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/EnhancedBigtableTableAdminStub.java index 0a6e8efec3..1cb80e0c49 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/EnhancedBigtableTableAdminStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/stub/EnhancedBigtableTableAdminStub.java @@ -31,6 +31,8 @@ import com.google.api.gax.rpc.UnaryCallable; import com.google.bigtable.admin.v2.OptimizeRestoredTableMetadata; import com.google.bigtable.admin.v2.TableName; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; +import com.google.cloud.bigtable.data.v2.internal.TableAdminRequestContext; import com.google.longrunning.Operation; import com.google.protobuf.Empty; import io.grpc.MethodDescriptor; @@ -52,27 +54,42 @@ public class EnhancedBigtableTableAdminStub extends GrpcBigtableTableAdminStub { private final BigtableTableAdminStubSettings settings; private final ClientContext clientContext; + private final TableAdminRequestContext requestContext; + private final AwaitReplicationCallable awaitReplicationCallable; + + private final AwaitConsistencyCallable awaitConsistencyCallable; private final OperationCallable optimizeRestoredTableOperationBaseCallable; public static EnhancedBigtableTableAdminStub createEnhanced( - BigtableTableAdminStubSettings settings) throws IOException { - return new EnhancedBigtableTableAdminStub(settings, ClientContext.create(settings)); + BigtableTableAdminStubSettings settings, TableAdminRequestContext requestContext) + throws IOException { + return new EnhancedBigtableTableAdminStub( + settings, ClientContext.create(settings), requestContext); } private EnhancedBigtableTableAdminStub( - BigtableTableAdminStubSettings settings, ClientContext clientContext) throws IOException { + BigtableTableAdminStubSettings settings, + ClientContext clientContext, + TableAdminRequestContext requestContext) + throws IOException { super(settings, clientContext); this.settings = settings; this.clientContext = clientContext; + this.requestContext = requestContext; + this.awaitConsistencyCallable = createAwaitConsistencyCallable(); this.awaitReplicationCallable = createAwaitReplicationCallable(); this.optimizeRestoredTableOperationBaseCallable = createOptimizeRestoredTableOperationBaseCallable(); } private AwaitReplicationCallable createAwaitReplicationCallable() { + return AwaitReplicationCallable.create(awaitConsistencyCallable); + } + + private AwaitConsistencyCallable createAwaitConsistencyCallable() { // TODO(igorbernstein2): expose polling settings RetrySettings pollingSettings = RetrySettings.newBuilder() @@ -92,11 +109,12 @@ private AwaitReplicationCallable createAwaitReplicationCallable() { .setRpcTimeoutMultiplier(1.0) .build(); - return AwaitReplicationCallable.create( + return AwaitConsistencyCallable.create( generateConsistencyTokenCallable(), checkConsistencyCallable(), clientContext, - pollingSettings); + pollingSettings, + requestContext); } // Plug into gax operation infrastructure @@ -194,6 +212,10 @@ public UnaryCallable awaitReplicationCallable() { return awaitReplicationCallable; } + public UnaryCallable awaitConsistencyCallable() { + return awaitConsistencyCallable; + } + public OperationCallable awaitOptimizeRestoredTableCallable() { return optimizeRestoredTableOperationBaseCallable; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/TableAdminRequestContext.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/TableAdminRequestContext.java new file mode 100644 index 0000000000..05554425b4 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/TableAdminRequestContext.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.internal; + +import com.google.api.core.InternalApi; +import com.google.auto.value.AutoValue; +import java.io.Serializable; + +/** + * Contains information necessary to construct Bigtable protobuf requests from user facing models. + * + *

The intention is to extract repetitive details like instance names into a configurable values + * in {@link com.google.cloud.bigtable.data.v2.BigtableDataSettings} and expose them (via this + * class) to each wrapper's toProto method. + * + *

This class is considered an internal implementation detail and not meant to be used by + * applications. + */ +@InternalApi +@AutoValue +public abstract class TableAdminRequestContext implements Serializable { + + /** Creates a new instance of the {@link TableAdminRequestContext}. */ + public static TableAdminRequestContext create(String projectId, String instanceId) { + return new AutoValue_TableAdminRequestContext(projectId, instanceId); + } + + /** The project id that the client is configured to target. */ + public abstract String getProjectId(); + + /** The instance id that the client is configured to target. */ + public abstract String getInstanceId(); +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java index a7f2f74a17..6cb830b11a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java @@ -63,6 +63,7 @@ import com.google.cloud.bigtable.admin.v2.internal.NameUtil; import com.google.cloud.bigtable.admin.v2.models.AuthorizedView; import com.google.cloud.bigtable.admin.v2.models.Backup; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; import com.google.cloud.bigtable.admin.v2.models.CopyBackupRequest; import com.google.cloud.bigtable.admin.v2.models.CreateAuthorizedViewRequest; import com.google.cloud.bigtable.admin.v2.models.CreateBackupRequest; @@ -156,6 +157,8 @@ public class BigtableTableAdminClientTests { @Mock private UnaryCallable mockDropRowRangeCallable; @Mock private UnaryCallable mockAwaitReplicationCallable; + @Mock private UnaryCallable mockAwaitConsistencyCallable; + @Mock private OperationCallable< com.google.bigtable.admin.v2.CreateBackupRequest, @@ -566,6 +569,30 @@ public void testAwaitReplication() { assertThat(wasCalled.get()).isTrue(); } + @Test + public void testAwaitConsistencyForDataBoost() { + // Setup + Mockito.when(mockStub.awaitConsistencyCallable()).thenReturn(mockAwaitConsistencyCallable); + + ConsistencyRequest consistencyRequest = ConsistencyRequest.forDataBoost(TABLE_ID); + + final AtomicBoolean wasCalled = new AtomicBoolean(false); + + Mockito.when(mockAwaitConsistencyCallable.futureCall(consistencyRequest)) + .thenAnswer( + (Answer>) + invocationOnMock -> { + wasCalled.set(true); + return ApiFutures.immediateFuture(null); + }); + + // Execute + adminClient.awaitConsistency(consistencyRequest); + + // Verify + assertThat(wasCalled.get()).isTrue(); + } + @Test public void testExistsTrue() { // Setup diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java index a1b5c97e34..cfcc8d0b42 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableTableAdminClientIT.java @@ -28,6 +28,7 @@ import com.google.cloud.Policy; import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient; import com.google.cloud.bigtable.admin.v2.models.ColumnFamily; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest; import com.google.cloud.bigtable.admin.v2.models.GCRules.DurationRule; import com.google.cloud.bigtable.admin.v2.models.GCRules.IntersectionRule; @@ -46,6 +47,7 @@ import org.junit.After; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -227,6 +229,23 @@ public void awaitReplication() { tableAdmin.awaitReplication(tableId); } + /** + * Note: Data Boost consistency is essentially a check that the data you are trying to read was + * written at least 35 minutes ago. The test thus takes ~35 minutes, and we should add a separate + * profile to run this concurrently with the other tests. + */ + @Test + @Ignore + public void awaitDataBoostConsistency() { + assume() + .withMessage("Data Boost consistency not supported on Emulator") + .that(testEnvRule.env()) + .isNotInstanceOf(EmulatorEnv.class); + tableAdmin.createTable(CreateTableRequest.of(tableId)); + ConsistencyRequest consistencyRequest = ConsistencyRequest.forDataBoost(tableId); + tableAdmin.awaitConsistency(consistencyRequest); + } + @Test public void iamUpdateTest() { assume() diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/ConsistencyRequestTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/ConsistencyRequestTest.java new file mode 100644 index 0000000000..cc039d7d80 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/models/ConsistencyRequestTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.admin.v2.models; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.bigtable.admin.v2.CheckConsistencyRequest; +import com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest; +import com.google.cloud.bigtable.data.v2.internal.TableAdminRequestContext; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ConsistencyRequestTest { + private final String PROJECT_ID = "my-project"; + private final String INSTANCE_ID = "my-instance"; + private final String TABLE_ID = "my-table"; + private final String CONSISTENCY_TOKEN = "my-token"; + + @Test + public void testToCheckConsistencyProtoWithStandard() { + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(TABLE_ID); + + TableAdminRequestContext requestContext = + TableAdminRequestContext.create(PROJECT_ID, INSTANCE_ID); + + CheckConsistencyRequest checkConsistencyRequest = + consistencyRequest.toCheckConsistencyProto(requestContext, CONSISTENCY_TOKEN); + + assertThat(checkConsistencyRequest.getName().equals(TABLE_ID)); + assertThat(checkConsistencyRequest.getConsistencyToken().equals(CONSISTENCY_TOKEN)); + assertThat( + checkConsistencyRequest + .getModeCase() + .equals(CheckConsistencyRequest.ModeCase.STANDARD_READ_REMOTE_WRITES)); + } + + @Test + public void testToCheckConsistencyProtoWithDataBoost() { + ConsistencyRequest consistencyRequest = ConsistencyRequest.forDataBoost(TABLE_ID); + + TableAdminRequestContext requestContext = + TableAdminRequestContext.create(PROJECT_ID, INSTANCE_ID); + + CheckConsistencyRequest checkConsistencyRequest = + consistencyRequest.toCheckConsistencyProto(requestContext, CONSISTENCY_TOKEN); + + assertThat(checkConsistencyRequest.getName().equals(TABLE_ID)); + assertThat(checkConsistencyRequest.getConsistencyToken().equals(CONSISTENCY_TOKEN)); + assertThat( + checkConsistencyRequest + .getModeCase() + .equals(CheckConsistencyRequest.ModeCase.DATA_BOOST_READ_LOCAL_WRITES)); + } + + @Test + public void testToGenerateTokenProto() { + ConsistencyRequest consistencyRequest = ConsistencyRequest.forDataBoost(TABLE_ID); + + TableAdminRequestContext requestContext = + TableAdminRequestContext.create(PROJECT_ID, INSTANCE_ID); + + GenerateConsistencyTokenRequest generateRequest = + consistencyRequest.toGenerateTokenProto(requestContext); + + assertThat(generateRequest.getName().equals(TABLE_ID)); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/stub/AwaitConsistencyCallableTest.java similarity index 62% rename from google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallableTest.java rename to google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/stub/AwaitConsistencyCallableTest.java index ac9941b2fc..2628cdf224 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/stub/AwaitReplicationCallableTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/stub/AwaitConsistencyCallableTest.java @@ -31,7 +31,10 @@ import com.google.bigtable.admin.v2.CheckConsistencyResponse; import com.google.bigtable.admin.v2.GenerateConsistencyTokenRequest; import com.google.bigtable.admin.v2.GenerateConsistencyTokenResponse; +import com.google.bigtable.admin.v2.StandardReadRemoteWrites; import com.google.bigtable.admin.v2.TableName; +import com.google.cloud.bigtable.admin.v2.models.ConsistencyRequest; +import com.google.cloud.bigtable.data.v2.internal.TableAdminRequestContext; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.Before; @@ -47,11 +50,16 @@ import org.threeten.bp.Duration; @RunWith(JUnit4.class) -public class AwaitReplicationCallableTest { +public class AwaitConsistencyCallableTest { @Rule public MockitoRule mockitoRule = MockitoJUnit.rule().strictness(Strictness.WARN); - private static final TableName TABLE_NAME = TableName.of("my-project", "my-instance", "my-table"); + private static final String PROJECT_ID = "my-project"; + private static final String INSTANCE_ID = "my-instance"; + private static final String TABLE_ID = "my-table"; + private static final TableName TABLE_NAME = TableName.of(PROJECT_ID, INSTANCE_ID, TABLE_ID); private static final ApiCallContext CALL_CONTEXT = FakeCallContext.createDefault(); + private static final TableAdminRequestContext REQUEST_CONTEXT = + TableAdminRequestContext.create(PROJECT_ID, INSTANCE_ID); @Mock private UnaryCallable @@ -61,7 +69,9 @@ public class AwaitReplicationCallableTest { private UnaryCallable mockCheckConsistencyCallable; - private AwaitReplicationCallable callable; + private AwaitReplicationCallable awaitReplicationCallable; + + private AwaitConsistencyCallable awaitConsistencyCallable; @Before public void setUp() { @@ -81,12 +91,14 @@ public void setUp() { .setRpcTimeoutMultiplier(1.0) .build(); - callable = - AwaitReplicationCallable.create( + awaitConsistencyCallable = + AwaitConsistencyCallable.create( mockGenerateConsistencyTokenCallable, mockCheckConsistencyCallable, clientContext, - retrySettings); + retrySettings, + REQUEST_CONTEXT); + awaitReplicationCallable = AwaitReplicationCallable.create(awaitConsistencyCallable); } @Test @@ -98,7 +110,8 @@ public void testGenerateFailure() throws Exception { Mockito.when(mockGenerateConsistencyTokenCallable.futureCall(expectedRequest, CALL_CONTEXT)) .thenReturn(ApiFutures.immediateFailedFuture(fakeError)); - ApiFuture future = callable.futureCall(TABLE_NAME, CALL_CONTEXT); + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(TABLE_ID); + ApiFuture future = awaitConsistencyCallable.futureCall(consistencyRequest, CALL_CONTEXT); Throwable actualError = null; @@ -125,6 +138,7 @@ public void testCheckFailure() throws Exception { CheckConsistencyRequest.newBuilder() .setName(TABLE_NAME.toString()) .setConsistencyToken("fake-token") + .setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()) .build(); FakeApiException expectedError = new FakeApiException("fake", null, Code.INTERNAL, false); @@ -132,7 +146,8 @@ public void testCheckFailure() throws Exception { Mockito.when(mockCheckConsistencyCallable.futureCall(expectedRequest2, CALL_CONTEXT)) .thenReturn(ApiFutures.immediateFailedFuture(expectedError)); - ApiFuture future = callable.futureCall(TABLE_NAME, CALL_CONTEXT); + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(TABLE_ID); + ApiFuture future = awaitConsistencyCallable.futureCall(consistencyRequest, CALL_CONTEXT); Throwable actualError = null; @@ -160,6 +175,7 @@ public void testImmediatelyConsistent() throws Exception { CheckConsistencyRequest.newBuilder() .setName(TABLE_NAME.toString()) .setConsistencyToken("fake-token") + .setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()) .build(); CheckConsistencyResponse expectedResponse2 = CheckConsistencyResponse.newBuilder().setConsistent(true).build(); @@ -167,7 +183,9 @@ public void testImmediatelyConsistent() throws Exception { Mockito.when(mockCheckConsistencyCallable.futureCall(expectedRequest2, CALL_CONTEXT)) .thenReturn(ApiFutures.immediateFuture(expectedResponse2)); - ApiFuture consistentFuture = callable.futureCall(TABLE_NAME, CALL_CONTEXT); + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(TABLE_ID); + ApiFuture consistentFuture = + awaitConsistencyCallable.futureCall(consistencyRequest, CALL_CONTEXT); consistentFuture.get(1, TimeUnit.MILLISECONDS); } @@ -187,6 +205,7 @@ public void testPolling() throws Exception { CheckConsistencyRequest.newBuilder() .setName(TABLE_NAME.toString()) .setConsistencyToken("fake-token") + .setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()) .build(); CheckConsistencyResponse expectedResponse2 = @@ -199,7 +218,9 @@ public void testPolling() throws Exception { .thenReturn(ApiFutures.immediateFuture(expectedResponse2)) .thenReturn(ApiFutures.immediateFuture(expectedResponse3)); - ApiFuture consistentFuture = callable.futureCall(TABLE_NAME, CALL_CONTEXT); + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(TABLE_ID); + ApiFuture consistentFuture = + awaitConsistencyCallable.futureCall(consistencyRequest, CALL_CONTEXT); consistentFuture.get(1, TimeUnit.SECONDS); } @@ -219,6 +240,7 @@ public void testPollingTimeout() throws Exception { CheckConsistencyRequest.newBuilder() .setName(TABLE_NAME.toString()) .setConsistencyToken("fake-token") + .setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()) .build(); CheckConsistencyResponse expectedResponse2 = @@ -227,7 +249,9 @@ public void testPollingTimeout() throws Exception { Mockito.when(mockCheckConsistencyCallable.futureCall(expectedRequest2, CALL_CONTEXT)) .thenReturn(ApiFutures.immediateFuture(expectedResponse2)); - ApiFuture consistentFuture = callable.futureCall(TABLE_NAME, CALL_CONTEXT); + ConsistencyRequest consistencyRequest = ConsistencyRequest.forReplication(TABLE_ID); + ApiFuture consistentFuture = + awaitConsistencyCallable.futureCall(consistencyRequest, CALL_CONTEXT); Throwable actualError = null; try { @@ -238,4 +262,67 @@ public void testPollingTimeout() throws Exception { assertThat(actualError).isInstanceOf(PollException.class); } + + @Test + public void testAwaitReplicationCallableImmediatelyConsistent() throws Exception { + GenerateConsistencyTokenRequest expectedRequest = + GenerateConsistencyTokenRequest.newBuilder().setName(TABLE_NAME.toString()).build(); + + GenerateConsistencyTokenResponse expectedResponse = + GenerateConsistencyTokenResponse.newBuilder().setConsistencyToken("fake-token").build(); + + Mockito.when(mockGenerateConsistencyTokenCallable.futureCall(expectedRequest, CALL_CONTEXT)) + .thenReturn(ApiFutures.immediateFuture(expectedResponse)); + + CheckConsistencyRequest expectedRequest2 = + CheckConsistencyRequest.newBuilder() + .setName(TABLE_NAME.toString()) + .setConsistencyToken("fake-token") + .setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()) + .build(); + CheckConsistencyResponse expectedResponse2 = + CheckConsistencyResponse.newBuilder().setConsistent(true).build(); + + Mockito.when(mockCheckConsistencyCallable.futureCall(expectedRequest2, CALL_CONTEXT)) + .thenReturn(ApiFutures.immediateFuture(expectedResponse2)); + + ApiFuture consistentFuture = + awaitReplicationCallable.futureCall(TABLE_NAME, CALL_CONTEXT); + + consistentFuture.get(1, TimeUnit.MILLISECONDS); + } + + @Test + public void testAwaitReplicationCallablePolling() throws Exception { + GenerateConsistencyTokenRequest expectedRequest = + GenerateConsistencyTokenRequest.newBuilder().setName(TABLE_NAME.toString()).build(); + + GenerateConsistencyTokenResponse expectedResponse = + GenerateConsistencyTokenResponse.newBuilder().setConsistencyToken("fake-token").build(); + + Mockito.when(mockGenerateConsistencyTokenCallable.futureCall(expectedRequest, CALL_CONTEXT)) + .thenReturn(ApiFutures.immediateFuture(expectedResponse)); + + CheckConsistencyRequest expectedRequest2 = + CheckConsistencyRequest.newBuilder() + .setName(TABLE_NAME.toString()) + .setConsistencyToken("fake-token") + .setStandardReadRemoteWrites(StandardReadRemoteWrites.newBuilder().build()) + .build(); + + CheckConsistencyResponse expectedResponse2 = + CheckConsistencyResponse.newBuilder().setConsistent(false).build(); + + CheckConsistencyResponse expectedResponse3 = + CheckConsistencyResponse.newBuilder().setConsistent(true).build(); + + Mockito.when(mockCheckConsistencyCallable.futureCall(expectedRequest2, CALL_CONTEXT)) + .thenReturn(ApiFutures.immediateFuture(expectedResponse2)) + .thenReturn(ApiFutures.immediateFuture(expectedResponse3)); + + ApiFuture consistentFuture = + awaitReplicationCallable.futureCall(TABLE_NAME, CALL_CONTEXT); + + consistentFuture.get(1, TimeUnit.SECONDS); + } }