chore: add experimental async data client (#920)

* feat: add new v3.0.0 API skeleton (#745)

* feat: improve rows filters (#751)

* feat: read rows query model class (#752)

* feat: implement row and cell model classes (#753)

* feat: add pooled grpc transport (#748)

* feat: implement read_rows (#762)

* feat: implement mutate rows (#769)

* feat: literal value filter (#767)

* feat: row_exists and read_row (#778)

* feat: read_modify_write and check_and_mutate_row (#780)

* feat: sharded read rows (#766)

* feat: ping and warm with metadata (#810)

* feat: mutate rows batching (#770)

* chore: restructure module paths (#816)

* feat: improve timeout structure (#819)

* fix: api errors apply to all bulk mutations

* chore: reduce public api surface (#820)

* feat: improve error group tracebacks on < py11 (#825)

* feat: optimize read_rows (#852)

* chore: add user agent suffix (#842)

* feat: optimize retries (#854)

* feat: add test proxy (#836)

* chore(tests): add conformance tests to CI for v3 (#870)

* chore(tests): turn off fast fail for conformance tets (#882)

* feat: add TABLE_DEFAULTS enum for table method arguments (#880)

* fix: pass None for retry in gapic calls (#881)

* feat: replace internal dictionaries with protos in gapic calls (#875)

* chore: optimize gapic calls (#863)

* feat: expose retryable error codes to users (#879)

* chore: update api_core submodule (#897)

* chore: merge main into experimental_v3 (#900)

* chore: pin conformance tests to v0.0.2 (#903)

* fix: bulk mutation eventual success (#909)

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

.coveragerc

@@ -22,7 +22,7 @@ omit =
   google/cloud/bigtable_admin/gapic_version.py
 
 [report]
-fail_under = 100
+fail_under = 99
 show_missing = True
 exclude_lines =
     # Re-enable the standard pragma

.github/sync-repo-settings.yaml

@@ -31,6 +31,24 @@ branchProtectionRules:
     - 'Kokoro'
     - 'Kokoro system-3.8'
     - 'cla/google'
+- pattern: experimental_v3
+  # Can admins overwrite branch protection.
+  # Defaults to `true`
+  isAdminEnforced: false
+  # Number of approving reviews required to update matching branches.
+  # Defaults to `1`
+  requiredApprovingReviewCount: 1
+  # Are reviews from code owners required to update matching branches.
+  # Defaults to `false`
+  requiresCodeOwnerReviews: false
+  # Require up to date branches
+  requiresStrictStatusChecks: false
+  # List of required status check contexts that must pass for commits to be accepted to matching branches.
+  requiredStatusCheckContexts:
+    - 'Kokoro'
+    - 'Kokoro system-3.8'
+    - 'cla/google'
+    - 'Conformance / Async v3 Client / Python 3.8'
 # List of explicit permissions to add (additive only)
 permissionRules:
     # Team slug to add to repository permissions

.github/workflows/conformance.yaml

@@ -0,0 +1,56 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Github action job to test core java library features on
+# downstream client libraries before they are released.
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+name: Conformance
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-version: [ "v0.0.2" ]
+        py-version: [ 3.8 ]
+        client-type: [ "Async v3", "Legacy" ]
+      fail-fast: false
+    name: "${{ matrix.client-type }} Client / Python ${{ matrix.py-version }} / Test Tag ${{ matrix.test-version }}"
+    steps:
+    - uses: actions/checkout@v3
+      name: "Checkout python-bigtable"
+    - uses: actions/checkout@v3
+      name: "Checkout conformance tests"
+      with:
+        repository: googleapis/cloud-bigtable-clients-test
+        ref: ${{ matrix.test-version }}
+        path: cloud-bigtable-clients-test
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.py-version }}
+    - uses: actions/setup-go@v4
+      with:
+        go-version: '>=1.20.2'
+    - run: chmod +x .kokoro/conformance.sh
+    - run: pip install -e .
+      name: "Install python-bigtable from HEAD"
+    - run: go version
+    - run: .kokoro/conformance.sh
+      name: "Run tests"
+      env:
+        CLIENT_TYPE: ${{ matrix.client-type }}
+        PYTHONUNBUFFERED: 1
+

.github/workflows/system_emulated.yml

@@ -20,7 +20,7 @@ jobs:
         python-version: '3.8'
 
     - name: Setup GCloud SDK
-      uses: google-github-actions/[email protected].1
+      uses: google-github-actions/[email protected].0
 
     - name: Install / run Nox
       run: |

.github/workflows/unittest.yml

@@ -54,4 +54,4 @@ jobs:
       run: |
         find .coverage-results -type f -name '*.zip' -exec unzip {} \;
         coverage combine .coverage-results/**/.coverage*
-        coverage report --show-missing --fail-under=100
+        coverage report --show-missing --fail-under=99

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "python-api-core"]
+	path = python-api-core
+	url = [email protected]:googleapis/python-api-core.git
+[submodule "gapic-generator-fork"]
+	path = gapic-generator-fork
+	url = [email protected]:googleapis/gapic-generator-python.git

.kokoro/conformance.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+## cd to the parent directory, i.e. the root of the git repo
+cd $(dirname $0)/..
+
+PROXY_ARGS=""
+TEST_ARGS=""
+if [[ "${CLIENT_TYPE^^}" == "LEGACY" ]]; then
+  echo "Using legacy client"
+  PROXY_ARGS="--legacy-client"
+  # legacy client does not expose mutate_row. Disable those tests
+  TEST_ARGS="-skip TestMutateRow_"
+fi
+
+# Build and start the proxy in a separate process
+PROXY_PORT=9999
+pushd test_proxy
+nohup python test_proxy.py --port $PROXY_PORT $PROXY_ARGS &
+proxyPID=$!
+popd
+
+# Kill proxy on exit
+function cleanup() {
+    echo "Cleanup testbench";
+    kill $proxyPID
+}
+trap cleanup EXIT
+
+# Run the conformance test
+pushd cloud-bigtable-clients-test/tests
+eval "go test -v -proxy_addr=:$PROXY_PORT $TEST_ARGS"
+RETURN_CODE=$?
+popd
+
+echo "exiting with ${RETURN_CODE}"
+exit ${RETURN_CODE}

.kokoro/presubmit/conformance.cfg

@@ -0,0 +1,6 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "NOX_SESSION"
+    value: "conformance"
+}

google/cloud/bigtable/data/__init__.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from google.cloud.bigtable import gapic_version as package_version
+
+from google.cloud.bigtable.data._async.client import BigtableDataClientAsync
+from google.cloud.bigtable.data._async.client import TableAsync
+
+from google.cloud.bigtable.data._async.mutations_batcher import MutationsBatcherAsync
+
+from google.cloud.bigtable.data.read_rows_query import ReadRowsQuery
+from google.cloud.bigtable.data.read_rows_query import RowRange
+from google.cloud.bigtable.data.row import Row
+from google.cloud.bigtable.data.row import Cell
+
+from google.cloud.bigtable.data.mutations import Mutation
+from google.cloud.bigtable.data.mutations import RowMutationEntry
+from google.cloud.bigtable.data.mutations import SetCell
+from google.cloud.bigtable.data.mutations import DeleteRangeFromColumn
+from google.cloud.bigtable.data.mutations import DeleteAllFromFamily
+from google.cloud.bigtable.data.mutations import DeleteAllFromRow
+
+from google.cloud.bigtable.data.exceptions import InvalidChunk
+from google.cloud.bigtable.data.exceptions import FailedMutationEntryError
+from google.cloud.bigtable.data.exceptions import FailedQueryShardError
+
+from google.cloud.bigtable.data.exceptions import RetryExceptionGroup
+from google.cloud.bigtable.data.exceptions import MutationsExceptionGroup
+from google.cloud.bigtable.data.exceptions import ShardedReadRowsExceptionGroup
+
+from google.cloud.bigtable.data._helpers import TABLE_DEFAULT
+from google.cloud.bigtable.data._helpers import RowKeySamples
+from google.cloud.bigtable.data._helpers import ShardedQuery
+
+
+__version__: str = package_version.__version__
+
+__all__ = (
+    "BigtableDataClientAsync",
+    "TableAsync",
+    "RowKeySamples",
+    "ReadRowsQuery",
+    "RowRange",
+    "MutationsBatcherAsync",
+    "Mutation",
+    "RowMutationEntry",
+    "SetCell",
+    "DeleteRangeFromColumn",
+    "DeleteAllFromFamily",
+    "DeleteAllFromRow",
+    "Row",
+    "Cell",
+    "InvalidChunk",
+    "FailedMutationEntryError",
+    "FailedQueryShardError",
+    "RetryExceptionGroup",
+    "MutationsExceptionGroup",
+    "ShardedReadRowsExceptionGroup",
+    "ShardedQuery",
+    "TABLE_DEFAULT",
+)

google/cloud/bigtable/data/_async/__init__.py

@@ -0,0 +1,25 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.cloud.bigtable.data._async.client import BigtableDataClientAsync
+from google.cloud.bigtable.data._async.client import TableAsync
+
+from google.cloud.bigtable.data._async.mutations_batcher import MutationsBatcherAsync
+
+
+__all__ = [
+    "BigtableDataClientAsync",
+    "TableAsync",
+    "MutationsBatcherAsync",
+]