From bb0dac5d5a8c642860cc4f61165fb0db88eb6ec4 Mon Sep 17 00:00:00 2001 From: riteshghorse Date: Tue, 19 Mar 2024 13:31:24 -0400 Subject: [PATCH] add vetex ai page --- .../python/elementwise/enrichment-bigtable.md | 63 +++++++++++++++++++ .../python/elementwise/enrichment-vertexai.md | 0 .../python/elementwise/enrichment.md | 37 ++++------- .../python/elementwise/runinference.md | 2 +- 4 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-bigtable.md create mode 100644 website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-vertexai.md diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-bigtable.md b/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-bigtable.md new file mode 100644 index 000000000000..d274df40a44b --- /dev/null +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-bigtable.md @@ -0,0 +1,63 @@ +--- +title: "Enrichment" +--- + + +# Enrichment transform + +{{< localstorage language language-py >}} + + + + + +
+ + {{< button-pydoc path="apache_beam.transforms" class="Enrichment" >}} + +
+ + +The following example demonstrates how to create a pipeline that does data enrichment with Cloud Bigtable. + +## Example: BigTableEnrichmentHandler + +The data stored in the Bigtable cluster uses the following format: + +{{ table }} +| Row key | product:product_id | product:product_name | product:product_stock | +|:---------:|:--------------------:|:----------------------:|:-----------------------:| +| 1 | 1 | pixel 5 | 2 | +| 2 | 2 | pixel 6 | 4 | +| 3 | 3 | pixel 7 | 20 | +| 4 | 4 | pixel 8 | 10 | +{{ /table }} + +{{< highlight language="py" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment.py" enrichment_with_bigtable >}} +{{}} + +{{< paragraph class="notebook-skip" >}} +Output: +{{< /paragraph >}} +{{< highlight class="notebook-skip" >}} +{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment_test.py" enrichment_with_bigtable >}} +{{< /highlight >}} + +## Related transforms + +Not applicable. + +{{< button-pydoc path="apache_beam.transforms" class="Enrichment" >}} \ No newline at end of file diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-vertexai.md b/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment-vertexai.md new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment.md b/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment.md index 5dfa5df04fae..fdd5c97e8cc9 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/enrichment.md @@ -23,7 +23,7 @@ limitations under the License. - {{< button-pydoc path="apache_beam.transforms" class="Enrichment" >}} + {{< button-pydoc path="apache_beam.transforms.enrichment" class="Enrichment" >}} @@ -31,33 +31,20 @@ limitations under the License. The enrichment transform lets you dynamically enrich data in a pipeline by doing a key-value lookup to a remote service. The transform uses [`RequestResponeIO`](https://beam.apache.org/releases/pydoc/current/apache_beam.io.requestresponseio.html#apache_beam.io.requestresponseio.RequestResponseIO) internally. This feature uses client-side throttling to ensure that the remote service isn't overloaded with requests. If service-side errors occur, like `TooManyRequests` and `Timeout` exceptions, it retries the requests by using exponential backoff. + +This transform is available in Apache Beam 2.54.0 and later versions. -In Apache Beam 2.54.0 and later versions, the transform includes a built-in enrichment handler for [Bigtable](https://cloud.google.com/bigtable/docs/overview). +## Examples -## Use Bigtable to enrich data +The following examples demonstrates how to create a pipeline that use the enrichment transform to enrich data from external services. -The following example demonstrates how to create a pipeline that use the enrichment transform with [`BigTableEnrichmentHandler`](https://beam.apache.org/releases/pydoc/current/apache_beam.transforms.enrichment_handlers.bigtable.html#apache_beam.transforms.enrichment_handlers.bigtable.BigTableEnrichmentHandler). - -The data stored in the Bigtable cluster uses the following format: - -| Row key | product:product_id | product:product_name | product:product_stock | -|:---------:|:--------------------:|:----------------------:|:-----------------------:| -| 1 | 1 | pixel 5 | 2 | -| 2 | 2 | pixel 6 | 4 | -| 3 | 3 | pixel 7 | 20 | -| 4 | 4 | pixel 8 | 10 | - - -{{< highlight language="py" >}} -{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment.py" enrichment_with_bigtable >}} -{{}} - -{{< paragraph class="notebook-skip" >}} -Output: -{{< /paragraph >}} -{{< highlight class="notebook-skip" >}} -{{< code_sample "sdks/python/apache_beam/examples/snippets/transforms/elementwise/enrichment_test.py" enrichment_with_bigtable >}} -{{< /highlight >}} +{{< table >}} +| Service | Beam version | Example | +|:----------------------------------------------|:-------------|:--------------------------------------------| +| Google Cloud Bigtable | \>= 2.54.0 | Bigtable Enrichment | +| Google Cloud Vertex AI Feature Store | \>= 2.55.0 | Vertex AI Feature Store Enrichment | +| Google Cloud Vertex AI Feature Store (Legacy) | \>= 2.55.0 | Vertex AI Feature Store (Legacy) Enrichment | +{{< /table >}} ## Related transforms diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md b/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md index 47944b9a232f..0f3cacf1d748 100644 --- a/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md +++ b/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md @@ -23,7 +23,7 @@ limitations under the License. - {{< button-pydoc path="apache_beam.ml.inference" class="RunInference" >}} + {{< button-pydoc path="apache_beam.ml.inference.base" class="RunInference" >}}