feat: add test proxy (#836)

.kokoro/presubmit/conformance.cfg

@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "NOX_SESSION"
+    value: "conformance"
+}

google/cloud/bigtable/data/_async/_mutate_rows.py

@@ -15,6 +15,7 @@
 from __future__ import annotations
 
 from typing import TYPE_CHECKING
+import asyncio
 import functools
 
 from google.api_core import exceptions as core_exceptions
@@ -183,6 +184,13 @@ async def _run_attempt(self):
                         self._handle_entry_error(orig_idx, entry_error)
                     # remove processed entry from active list
                     del active_request_indices[result.index]
+        except asyncio.CancelledError:
+            # when retry wrapper timeout expires, the operation is cancelled
+            # make sure incomplete indices are tracked,
+            # but don't record exception (it will be raised by wrapper)
+            # TODO: remove asyncio.wait_for in retry wrapper. Let grpc call handle expiration
+            self.remaining_indices.extend(active_request_indices.values())
+            raise
         except Exception as exc:
             # add this exception to list for each mutation that wasn't
             # already handled, and update remaining_indices if mutation is retryable

google/cloud/bigtable/data/_async/client.py

@@ -30,6 +30,7 @@
 import warnings
 import sys
 import random
+import os
 
 from collections import namedtuple
 
@@ -38,10 +39,12 @@
 from google.cloud.bigtable_v2.services.bigtable.async_client import DEFAULT_CLIENT_INFO
 from google.cloud.bigtable_v2.services.bigtable.transports.pooled_grpc_asyncio import (
     PooledBigtableGrpcAsyncIOTransport,
+    PooledChannel,
 )
 from google.cloud.bigtable_v2.types.bigtable import PingAndWarmRequest
 from google.cloud.client import ClientWithProject
 from google.api_core.exceptions import GoogleAPICallError
+from google.cloud.environment_vars import BIGTABLE_EMULATOR  # type: ignore
 from google.api_core import retry_async as retries
 from google.api_core import exceptions as core_exceptions
 from google.cloud.bigtable.data._async._read_rows import _ReadRowsOperationAsync
@@ -150,18 +153,35 @@ def __init__(
         # keep track of table objects associated with each instance
         # only remove instance from _active_instances when all associated tables remove it
         self._instance_owners: dict[_WarmedInstanceKey, Set[int]] = {}
-        # attempt to start background tasks
         self._channel_init_time = time.monotonic()
         self._channel_refresh_tasks: list[asyncio.Task[None]] = []
-        try:
-            self._start_background_channel_refresh()
-        except RuntimeError:
+        self._emulator_host = os.getenv(BIGTABLE_EMULATOR)
+        if self._emulator_host is not None:
+            # connect to an emulator host
             warnings.warn(
-                f"{self.__class__.__name__} should be started in an "
-                "asyncio event loop. Channel refresh will not be started",
+                "Connecting to Bigtable emulator at {}".format(self._emulator_host),
                 RuntimeWarning,
                 stacklevel=2,
             )
+            self.transport._grpc_channel = PooledChannel(
+                pool_size=pool_size,
+                host=self._emulator_host,
+                insecure=True,
+            )
+            # refresh cached stubs to use emulator pool
+            self.transport._stubs = {}
+            self.transport._prep_wrapped_messages(client_info)
+        else:
+            # attempt to start background channel refresh tasks
+            try:
+                self._start_background_channel_refresh()
+            except RuntimeError:
+                warnings.warn(
+                    f"{self.__class__.__name__} should be started in an "
+                    "asyncio event loop. Channel refresh will not be started",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
 
     @staticmethod
     def _client_version() -> str:
@@ -176,7 +196,7 @@ def _start_background_channel_refresh(self) -> None:
         Raises:
           - RuntimeError if not called in an asyncio event loop
         """
-        if not self._channel_refresh_tasks:
+        if not self._channel_refresh_tasks and not self._emulator_host:
             # raise RuntimeError if there is no event loop
             asyncio.get_running_loop()
             for channel_idx in range(self.transport.pool_size):

noxfile.py

@@ -278,6 +278,27 @@ def system_emulated(session):
         os.killpg(os.getpgid(p.pid), signal.SIGKILL)
 
 
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+def conformance(session):
+    """
+    Run the set of shared bigtable conformance tests
+    """
+    TEST_REPO_URL = "https://github.com/googleapis/cloud-bigtable-clients-test.git"
+    CLONE_REPO_DIR = "cloud-bigtable-clients-test"
+    # install dependencies
+    constraints_path = str(
+        CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+    )
+    install_unittest_dependencies(session, "-c", constraints_path)
+    with session.chdir("test_proxy"):
+        # download the conformance test suite
+        clone_dir = os.path.join(CURRENT_DIRECTORY, CLONE_REPO_DIR)
+        if not os.path.exists(clone_dir):
+            print("downloading copy of test repo")
+            session.run("git", "clone", TEST_REPO_URL, CLONE_REPO_DIR, external=True)
+        session.run("bash", "-e", "run_tests.sh", external=True)
+
+
 @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system(session):
     """Run the system test suite."""

test_proxy/README.md

@@ -0,0 +1,60 @@
+# CBT Python Test Proxy
+
+The CBT test proxy is intended for running conformance tests for Cloud Bigtable Python Client.
+
+## Option 1: Run Tests with Nox
+
+You can run the conformance tests in a single line by calling `nox -s conformance` from the repo root
+
+
+```
+cd python-bigtable/test_proxy
+nox -s conformance
+```
+
+## Option 2: Run processes manually
+
+### Start test proxy
+
+You can use `test_proxy.py` to launch a new test proxy process directly
+
+```
+cd python-bigtable/test_proxy
+python test_proxy.py
+```
+
+The port can be set by passing in an extra positional argument
+
+```
+cd python-bigtable/test_proxy
+python test_proxy.py --port 8080
+```
+
+You can run the test proxy against the previous `v2` client by running it with the `--legacy-client` flag:
+
+```
+python test_proxy.py --legacy-client
+```
+
+### Run the test cases
+
+Prerequisites:
+- If you have not already done so, [install golang](https://go.dev/doc/install).
+- Before running tests, [launch an instance of the test proxy](#start-test-proxy) 
+in a separate shell session, and make note of the port
+
+
+Clone and navigate to the go test library:
+
+```
+git clone https://github.com/googleapis/cloud-bigtable-clients-test.git
+cd cloud-bigtable-clients-test/tests
+```
+
+
+Launch the tests
+
+```
+go test -v -proxy_addr=:50055
+```
+