diff --git a/.asf.yaml b/.asf.yaml
index ef141f3bb3fa..d126c562838f 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -49,6 +49,7 @@ github:
 
   protected_branches:
     master: {}
+    release-2.53.0: {}
     release-2.52.0: {}
     release-2.51.0: {}
     release-2.50.0: {}
diff --git a/.github/REVIEWERS.yml b/.github/REVIEWERS.yml
index c2a57ac8feef..b878d2f81cc8 100644
--- a/.github/REVIEWERS.yml
+++ b/.github/REVIEWERS.yml
@@ -60,6 +60,10 @@ labels:
       - igorbernstein2
       - mutianf
     exclusionList: []
+  - name: healthcare
+    reviewers:
+      - svetakvsundhar
+    exclusionList: []
   - name: Build
     reviewers:
       - damccorm
diff --git a/.github/actions/gradle-command-action b/.github/actions/gradle-command-action
deleted file mode 160000
index 90ccf054e6b9..000000000000
--- a/.github/actions/gradle-command-action
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 90ccf054e6b9905f30f98c938bce4c6acd323b6b
diff --git a/.github/actions/gradle-command-self-hosted-action/action.yml b/.github/actions/gradle-command-self-hosted-action/action.yml
index 906b35169d9d..e2fd768220a3 100644
--- a/.github/actions/gradle-command-self-hosted-action/action.yml
+++ b/.github/actions/gradle-command-self-hosted-action/action.yml
@@ -35,9 +35,9 @@ runs:
     - name: Run Gradle Command
       shell: bash
       run: |
-        # Removing settings.xml is a workaround to avoid a decryption issue
-        # of Beam's gradle-command-action plugin and github's provided
-        # maven settings.xml file
+        # This step is a workaround to avoid a decryption issue of Beam's
+        # net.linguica.gradle.maven.settings plugin and github's provided maven
+        # settings.xml file
         if [ -f ~/.m2/settings.xml ]; then
           rm ~/.m2/settings.xml
         fi
diff --git a/.github/actions/setup-action/action.yml b/.github/actions/setup-action/action.yml
index 743e89a931da..4c4bb2753194 100644
--- a/.github/actions/setup-action/action.yml
+++ b/.github/actions/setup-action/action.yml
@@ -36,8 +36,10 @@ runs:
       shell: bash
       if: github.event_name == 'issue_comment' && github.event.comment.body != inputs.comment_phrase
       run: |
-        echo "The comment ${{ github.event.comment.body }} does not match the phrase for this instance:  ${{ inputs.comment_phrase }}. Exiting."
+        echo "The comment $PHRASE does not match the phrase for this instance:  ${{ inputs.comment_phrase }}. Exiting."
         exit 1
+      env:
+        PHRASE: "${{ github.event.comment.body }}"
     - name: Check out repository code if pull request commit
       shell: bash
       if: ${{ github.event_name == 'pull_request_target' }}
@@ -69,4 +71,4 @@ runs:
     - name: expose gcloud path
       shell: bash
       run: |
-        echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV
\ No newline at end of file
+        echo KUBELET_GCLOUD_CONFIG_PATH=/var/lib/kubelet/pods/$POD_UID/volumes/kubernetes.io~empty-dir/gcloud >> $GITHUB_ENV
diff --git a/.github/autolabeler.yml b/.github/autolabeler.yml
index 57c8f65c6ac1..a4ce41fce073 100644
--- a/.github/autolabeler.yml
+++ b/.github/autolabeler.yml
@@ -57,6 +57,7 @@ io:  ["sdks/go/pkg/beam/io/**/*", "sdks/java/io/**/*", "sdks/python/apache_beam/
 "hadoop":  ["sdks/java/io/hadoop-common/**/*", "sdks/java/io/hadoop-file-system/**/*", "sdks/java/io/hadoop-format/**/*"]
 "hbase":  ["sdks/java/io/hbase/**/*"]
 "hcatalog":  ["sdks/java/io/hcatalog/**/*"]
+"healthcare":  ["sdks/python/apache_beam/io/gcp/healthcare/**/*", "sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/healthcare/**/*"]
 "jdbc":  ["sdks/java/io/jdbc/**/*"]
 "jms":  ["sdks/java/io/jms/**/*"]
 "kafka":  ["sdks/java/io/kafka/**/*"]
diff --git a/.github/build.gradle b/.github/build.gradle
index acfe7f3686a9..de2670545625 100644
--- a/.github/build.gradle
+++ b/.github/build.gradle
@@ -28,6 +28,7 @@ buildscript {
 /** check that yml are valid */
 task check {
   doLast {
+    List<String> errors = []
     fileTree("${project.projectDir}/workflows").matching {
       include "*.yml"
       include "*.yaml"
@@ -36,23 +37,41 @@ task check {
       // attempt load yml to make sure its valid
       def workflow = new org.yaml.snakeyaml.Yaml().load(it.newInputStream())
 
-      // additional guards for running all tests functionality
-      // TODO(yathu) expand this also to post commits prior teardown Jenkins postcommits
-      if ( fname.startsWith("beam_PreCommit")) {
+      // additional guards to ensure tests configured in same way
+      if ( fname.startsWith("beam_PreCommit") || fname.startsWith("beam_PostCommit") ) {
         List paths
         try {
           paths = workflow.getAt(true).pull_request_target.paths as List
         } catch (Exception e) {
-          throw new GradleException("Fail to get the trigger path for ${fname}. " +
-                  "Make sure precommit has a pull_request_target trigger.", e)
+          errors.add("Fail to get the trigger path for ${fname}. " +
+                  "Make sure it has a pull_request_target trigger.")
+          return
         }
-        // precommit should triggered by this specific file
+
+        // precommit and postcommit should triggered by this specific file
+        // this is to ensure not missing test during release branch verification
         if (paths != null && !paths.contains('release/trigger_all_tests.json')) {
-          throw new GradleException("Error validating ${fname}: " +
-                  "Please add 'release/trigger_all_tests.json' to the trigger path for release verification run properly")
+          errors.add("Error validating ${fname}: " +
+                  "Please add 'release/trigger_all_tests.json' to the trigger path")
+          return
+        }
+
+        // postcommit should triggered by a specific file so that there is a way to exercise post for open PR
+        // TODO(https://github.com/apache/beam/issues/28909)
+        //  remove file match trigger once a better trigger (e.g. comment trigger) is implemented
+        if (fname.startsWith("beam_PostCommit")) {
+          String triggerFile = '.github/trigger_files/' + fname.take(fname.lastIndexOf('.')) + '.json'
+          if (paths != null && !paths.contains(triggerFile)) {
+            errors.add("Error validating ${fname}: " +
+                    "Please add ${triggerFile} to the trigger path")
+            return
+          }
         }
       }
     }
+    if (!errors.isEmpty()) {
+      throw new GradleException("Check failed: " + errors.join('\n'))
+    }
   }
 }
 
diff --git a/.github/gh-actions-self-hosted-runners/arc/locals.tf b/.github/gh-actions-self-hosted-runners/arc/locals.tf
index a69d069ab865..7820ce3e8aa3 100644
--- a/.github/gh-actions-self-hosted-runners/arc/locals.tf
+++ b/.github/gh-actions-self-hosted-runners/arc/locals.tf
@@ -20,6 +20,8 @@
 
 locals {
         arc_values = {
+            #fix dind issue
+            "image.dindSidecarRepositoryAndTag" = "docker:24.0.7-dind-alpine3.18"
             "githubWebhookServer.enabled" = "${var.deploy_webhook}"
             "authSecret.create" = "true"
             "authSecret.github_app_id" = data.google_secret_manager_secret_version.github_app_id.secret_data
diff --git a/.github/trigger_files/beam_PostCommit_Go.json b/.github/trigger_files/beam_PostCommit_Go.json
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/.github/trigger_files/beam_PostCommit_Go_VR_Flink.json b/.github/trigger_files/beam_PostCommit_Go_VR_Flink.json
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/.github/trigger_files/beam_PostCommit_Go_VR_Samza.json b/.github/trigger_files/beam_PostCommit_Go_VR_Samza.json
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/.github/trigger_files/beam_PostCommit_Go_VR_Spark.json b/.github/trigger_files/beam_PostCommit_Go_VR_Spark.json
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index c4f437dedd29..b39dfc9d5924 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -138,6 +138,8 @@ In order to make it easier for non-committers to interact with workflows, workfl
 2) Each job should have the rerun action immediately after its checkout step. You can add a step that uses the `setup-action` action in your workflow, which encapsulates the checkout and rerun logic in one place. This should be gated on the comment trigger (example: https://github.com/apache/beam/blob/0ee2dc73ec6f555a5bf1a643dffd37f4927be67e/.github/workflows/beam_PreCommit_Go.yml#L65-L70)
 3) Each job should have a descriptive name that includes the comment trigger (example: https://github.com/apache/beam/blob/ba8fc935222aeb070668fbafd588bc58e7a21289/.github/workflows/beam_PreCommit_CommunityMetrics.yml#L48)
 
+**Note:** this approach is found not scalable ([#28909](https://github.com/apache/beam/issues/28909)) and currently only enabled for PreCommit workflows. For PostCommit jobs, it is currently replaced by a temporary approach of `pull_request_target` trigger with specific path `.github/trigger_files/<workflow_file_name_stem.json>`.
+
 # Testing new workflows or workflow updates
 
 ## Testing New Workflows
@@ -192,6 +194,8 @@ Please note that jobs with matrix need to have matrix element in the comment. Ex
 
 ### PreCommit Jobs
 
+PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sources has changed. To manually trigger certain PreCommit job, comment with the Trigger Phrase (listed below) in the PR.
+
 | Workflow name | Matrix | Trigger Phrase | Cron Status |
 |:-------------:|:------:|:--------------:|:-----------:|
 | [ PreCommit Community Metrics ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_CommunityMetrics.yml) | N/A |`Run CommunityMetrics PreCommit`| [![.github/workflows/beam_PreCommit_CommunityMetrics.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_CommunityMetrics.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_CommunityMetrics.yml?query=event%3Aschedule) |
@@ -269,207 +273,210 @@ Please note that jobs with matrix need to have matrix element in the comment. Ex
 
 ### PostCommit Jobs
 
-| Workflow name | Matrix | Trigger Phrase | Cron Status |
+PostCommit Jobs run in a schedule against master branch and generally do not get triggered in a PR. To manually trigger certain PostCommit job, attach a file named "Trigger file" (listed below) under `.github/trigger_files/` folder.
+
+| Workflow name | Matrix | Trigger file | Cron Status |
 |:-------------:|:------:|:--------------:|:-----------:|
-| [ PostCommit BeamMetrics Publish ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_BeamMetrics_Publish.yml) | N/A |`Run Beam Metrics Deployment`| [![.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_BeamMetrics_Publish.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_BeamMetrics_Publish.yml?query=event%3Aschedule)
-| [ PostCommit Go ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml) | N/A |`Run Go PostCommit`| [![.github/workflows/beam_PostCommit_Go.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml?query=event%3Aschedule) |
-| [ PostCommit Go Dataflow ARM](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml) | N/A |`Run Go PostCommit Dataflow ARM`| [![.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml?query=event%3Aschedule) |
-| [ PostCommit Go VR Flink](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml) | N/A |`Run Go Flink ValidatesRunner`| [![.github/workflows/beam_PostCommit_Go_VR_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Go VR Samza](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml) | N/A |`Run Go Samza ValidatesRunner`| [![.github/workflows/beam_PostCommit_Go_VR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml?query=event%3Aschedule) |
-| [ PostCommit Go VR Spark](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml) | N/A |`Run Go Spark ValidatesRunner`| [![.github/workflows/beam_PostCommit_Go_VR_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Java Avro Versions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml) | N/A |`Run Java Avro Versions PostCommit`| [![.github/workflows/beam_PostCommit_Java_Avro_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml?query=event%3Aschedule) |
-| [ PostCommit Java Dataflow V1 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) | N/A |`Run PostCommit_Java_Dataflow`| [![.github/workflows/beam_PostCommit_Java_DataflowV1.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml?query=event%3Aschedule) |
-| [ PostCommit Java Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | N/A |`Run PostCommit_Java_DataflowV2`| [![.github/workflows/beam_PostCommit_Java_DataflowV2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | ['8','11','17','21'] |`Run Java_Examples_Dataflow_ARM PostCommit (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java examples on Dataflow`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['11','17','21'] |`Run Java examples on Dataflow Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml) | N/A |`Run Java Examples on Dataflow Runner V2`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml) | ['11','17','21'] |`Run Java (matrix_element) Examples on Dataflow Runner V2`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) | N/A |`Run Java Examples_Direct`| [![.github/workflows/beam_PostCommit_Java_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) | N/A |`Run Java Examples_Flink`| [![.github/workflows/beam_PostCommit_Java_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Java Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml) | N/A |`Run Java Examples_Spark`| [![.github/workflows/beam_PostCommit_Java_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Java Hadoop Versions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml) | N/A |`Run PostCommit_Java_Hadoop_Versions`| [![.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml?query=event%3Aschedule) |
-| [ PostCommit Java InfluxDbIO Integration Test ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml) | N/A |`Run Java InfluxDbIO_IT`| [![.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml?query=event%3Aschedule)
-| [ PostCommit Java Jpms Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml) | N/A |`Run Jpms Dataflow Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml?query=event%3Aschedule) |
-| [ PostCommit Java Jpms Dataflow Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml) | N/A |`Run Jpms Dataflow Java 17 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml?query=event%3Aschedule) |
-| [ PostCommit Java Jpms Direct Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml) | N/A |`Run Jpms Direct Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml?query=event%3Aschedule) |
-| [ PostCommit Java Jpms Direct Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml) | N/A |`Run Jpms Direct Java 17 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml?query=event%3Aschedule) |
-| [ PostCommit Java Jpms Direct Java21 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml) | N/A |`Run Jpms Direct Java21 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml?query=event%3Aschedule) |
-| [ PostCommit Java Jpms Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml) | N/A |`Run Jpms Flink Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml?query=event%3Aschedule) |
-| [ PostCommit Java Jpms Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml) | N/A |`Run Jpms Spark Java 11 PostCommit`| [![.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml?query=event%3Aschedule) |
-| [ PostCommit Java Nexmark Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml) | N/A |`Run Dataflow Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Java Nexmark Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml) | N/A |`Run Dataflow Runner V2 Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml?query=event%3Aschedule) |
-| [ PostCommit Java Nexmark Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml) | ['11','17'] |`Run Dataflow Runner V2 Java (matrix) Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml?query=event%3Aschedule) |
-| [ PostCommit Java Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml) | N/A |`Run Direct Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit Java Nexmark Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml) | N/A |`Run Flink Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Java Nexmark Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml) | N/A |`Run Spark Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Java PVR Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml) | N/A |`Run Java Flink PortableValidatesRunner Streaming`| [![.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml?query=event%3Aschedule) |
-| [ PostCommit Java PVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml) | N/A |`Run Java Samza PortableValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_PVR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml?query=event%3Aschedule) |
-| [ PostCommit Java SingleStoreIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml) | N/A |`Run Java SingleStoreIO_IT`| [![.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml?query=event%3Aschedule) |
-| [ PostCommit Java PVR Spark3 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml) | N/A |`Run Java Spark v3 PortableValidatesRunner Streaming`| [![.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml?query=event%3Aschedule) |
-| [ PostCommit Java PVR Spark Batch ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml) | N/A |`Run Java Spark PortableValidatesRunner Batch`| [![.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml?query=event%3Aschedule) |
-| [ PostCommit Java Sickbay ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml) | N/A |`Run Java Sickbay`| [![.github/workflows/beam_PostCommit_Java_Sickbay.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml?query=event%3Aschedule) |
-| [ PostCommit Java Tpcds Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml) | N/A |`Run Dataflow Runner Tpcds Tests`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Java Tpcds Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml) | N/A |`Run Flink Runner Tpcds Tests`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Java Tpcds Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml) | N/A |`Run Spark Runner Tpcds Tests`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Dataflow JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml) | ['11','17'] |`Run Dataflow ValidatesRunner Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml) | N/A |`Run Dataflow Streaming ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Dataflow V2 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml) | N/A |`Run Java Dataflow V2 ValidatesRunner Streaming`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml) | N/A |`Run Java Dataflow V2 ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml) | N/A |`Run Dataflow ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Direct JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml) | ['11','17'] |`Run Direct ValidatesRunner Java (matrix_element)`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml) | N/A |`Run Direct ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml) | N/A |`Run Flink ValidatesRunner Java 11`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml) | N/A |`Run Flink ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml) | N/A |`Run Samza ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml) | N/A |`Run Spark ValidatesRunner Java 11`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml) | N/A |`Run Spark ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner SparkStructuredStreaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml) | N/A |`Run Spark StructuredStreaming ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner Twister2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml) | N/A |`Run Twister2 ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml?query=event%3Aschedule) |
-| [ PostCommit Java ValidatesRunner ULR ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml) | N/A |`Run ULR Loopback ValidatesRunner`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml?query=event%3Aschedule) |
-| [ PostCommit Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml) | N/A |`Run Java PostCommit`| [![.github/workflows/beam_PostCommit_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml?query=event%3Aschedule) |
-| [ PostCommit Javadoc ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml) | N/A |`Run Javadoc PostCommit`| [![.github/workflows/beam_PostCommit_Javadoc.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml?query=event%3Aschedule) |
-| [ PostCommit PortableJar Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml) | N/A |`Run PortableJar_Flink PostCommit`| [![.github/workflows/beam_PostCommit_PortableJar_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit PortableJar Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml) | N/A |`Run PortableJar_Spark PostCommit`| [![.github/workflows/beam_PostCommit_PortableJar_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python PostCommit (matrix_element)`| [![.github/workflows/beam_PostCommit_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml?query=event%3Aschedule) |
-| [ PostCommit Python Arm](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python PostCommit Arm (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Arm.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml?query=event%3Aschedule) |
-| [ PostCommit Python Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml) | N/A |`Run Python Examples_Dataflow`| [![.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Python Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python Examples_Direct (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit Python Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml) | ['3.8','3.11'] |`Run Python Examples_Flink (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Python Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml) | ['3.8','3.11'] |`Run Python Examples_Spark (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Python MongoDBIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml) | N/A |`Run Python MongoDBIO_IT`| [![.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml?query=event%3Aschedule) |
-| [ PostCommit Python Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml) | N/A |`Run Python Direct Runner Nexmark Tests`| [![.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit Python ValidatesContainer Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python Dataflow ValidatesContainer (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Python ValidatesContainer Dataflow With RC ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python RC Dataflow ValidatesContainer (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml?query=event%3Aschedule) |
-| [ PostCommit Python ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml) | ['3.8','3.11'] |`Run Python Dataflow ValidatesRunner (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Python ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml) | ['3.8','3.11'] |`Run Python Flink ValidatesRunner (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit Python ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml) | ['3.8','3.11'] |`Run Python Samza ValidatesRunner (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml?query=event%3Aschedule) |
-| [ PostCommit Python ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml) | ['3.8','3.9','3.11'] |`Run Python Spark ValidatesRunner (matrix_element)`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml?query=event%3Aschedule) |
-| [ PostCommit Python Xlang Gcp Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml) | N/A |`Run Python_Xlang_Gcp_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Python Xlang Gcp Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml) | N/A |`Run Python_Xlang_Gcp_Direct PostCommit`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit Python Xlang IO Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml) | N/A |`Run Python_Xlang_IO_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit Sickbay Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml) | ['3.8','3.9','3.10','3.11']  |`Run Python (matrix_element) PostCommit Sickbay`| [![.github/workflows/beam_PostCommit_Sickbay_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml?query=event%3Aschedule) |
-| [ PostCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml) | N/A |`Run SQL PostCommit`| [![.github/workflows/beam_PostCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml?query=event%3Aschedule) |
-| [ PostCommit TransformService Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml) | N/A |`Run TransformService_Direct PostCommit`| [![.github/workflows/beam_PostCommit_TransformService_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml?query=event%3Aschedule)
-| [ PostCommit Website Publish ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Publish.yml) | N/A | N/A | [![.github/workflows/beam_PostCommit_Website_Publish.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Publish.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Publish.yml?query=event%3Aschedule) |
-| [ PostCommit Website Test](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml) | N/A |`Run Full Website Test`| [![.github/workflows/beam_PostCommit_Website_Test.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml?query=event%3Aschedule) |
-| [ PostCommit XVR GoUsingJava Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml) | N/A |`Run XVR_GoUsingJava_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit XVR Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml) | N/A |`Run XVR_Direct PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml?query=event%3Aschedule) |
-| [ PostCommit XVR Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml) | N/A |`Run XVR_Flink PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml?query=event%3Aschedule) |
-| [ PostCommit XVR JavaUsingPython Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml) | N/A |`Run XVR_JavaUsingPython_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit XVR PythonUsingJava Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml) | N/A |`Run XVR_PythonUsingJava_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit XVR PythonUsingJavaSQL Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml) | N/A |`Run XVR_PythonUsingJavaSQL_Dataflow PostCommit`| [![.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml?query=event%3Aschedule) |
-| [ PostCommit XVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml) | N/A |`Run XVR_Samza PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml?query=event%3Aschedule) |
-| [ PostCommit XVR Spark3 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml) | N/A |`Run XVR_Spark3 PostCommit`| [![.github/workflows/beam_PostCommit_XVR_Spark3.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml?query=event%3Aschedule) |
-| [ Python Validates Container Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml) | ['3.8','3.9','3.10','3.11'] |`Run Python ValidatesContainer Dataflow ARM (matrix_element)`|[![.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml?query=event%3Aschedule) |
+| [ PostCommit Go ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml) | N/A |`beam_PostCommit_Go.json`| [![.github/workflows/beam_PostCommit_Go.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go.yml?query=event%3Aschedule) |
+| [ PostCommit Go Dataflow ARM](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml) | N/A |`beam_PostCommit_Go_Dataflow_ARM.json`| [![.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_Dataflow_ARM.yml?query=event%3Aschedule) |
+| [ PostCommit Go VR Flink](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml) | N/A |`beam_PostCommit_Go_VR_Flink.json`| [![.github/workflows/beam_PostCommit_Go_VR_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Go VR Samza](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml) | N/A |`beam_PostCommit_Go_VR_Samza.json`| [![.github/workflows/beam_PostCommit_Go_VR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Samza.yml?query=event%3Aschedule) |
+| [ PostCommit Go VR Spark](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml) | N/A |`beam_PostCommit_Go_VR_Spark.json`| [![.github/workflows/beam_PostCommit_Go_VR_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Go_VR_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Java Avro Versions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml) | N/A |`beam_PostCommit_Java_Avro_Versions.json`| [![.github/workflows/beam_PostCommit_Java_Avro_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Avro_Versions.yml?query=event%3Aschedule) |
+| [ PostCommit Java Dataflow V1 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml) | N/A |`beam_PostCommit_Java_DataflowV1.json`| [![.github/workflows/beam_PostCommit_Java_DataflowV1.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV1.yml?query=event%3Aschedule) |
+| [ PostCommit Java Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | N/A |`beam_PostCommit_Java_DataflowV2.json`| [![.github/workflows/beam_PostCommit_Java_DataflowV2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | ['8','11','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_ARM.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | N/A |`beam_PostCommit_Java_Examples_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['11','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml) | N/A |`beam_PostCommit_Java_Examples_Dataflow_V2.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml) | ['11','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_V2_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) | N/A |`beam_PostCommit_Java_Examples_Direct.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) | N/A |`beam_PostCommit_Java_Examples_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Java Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml) | N/A |`beam_PostCommit_Java_Examples_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Java Hadoop Versions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml) | N/A |`beam_PostCommit_Java_Hadoop_Versions.json`| [![.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Hadoop_Versions.yml?query=event%3Aschedule) |
+| [ PostCommit Java IO Performance Tests ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml) | N/A |`beam_PostCommit_Java_IO_Performance_Tests.json`| [![.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml?query=event%3Aschedule)
+| [ PostCommit Java InfluxDbIO Integration Test ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml) | N/A |`beam_PostCommit_Java_InfluxDbIO_IT.json`| [![.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml?query=event%3Aschedule)
+| [ PostCommit Java Jpms Dataflow Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml) | N/A |`beam_PostCommit_Java_Jpms_Dataflow_Java11.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml?query=event%3Aschedule) |
+| [ PostCommit Java Jpms Dataflow Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml) | N/A |`beam_PostCommit_Java_Jpms_Dataflow_Java17.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml?query=event%3Aschedule) |
+| [ PostCommit Java Jpms Direct Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml) | N/A |`beam_PostCommit_Java_Jpms_Direct_Java11.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml?query=event%3Aschedule) |
+| [ PostCommit Java Jpms Direct Java17 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml) | N/A |`beam_PostCommit_Java_Jpms_Direct_Java17.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml?query=event%3Aschedule) |
+| [ PostCommit Java Jpms Direct Java21 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml) | N/A |`beam_PostCommit_Java_Jpms_Direct_Java21.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml?query=event%3Aschedule) |
+| [ PostCommit Java Jpms Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml) | N/A |`beam_PostCommit_Java_Jpms_Flink_Java11.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml?query=event%3Aschedule) |
+| [ PostCommit Java Jpms Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml) | N/A |`beam_PostCommit_Java_Jpms_Spark_Java11.json`| [![.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml?query=event%3Aschedule) |
+| [ PostCommit Java Nexmark Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml) | N/A |`beam_PostCommit_Java_Nexmark_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Java Nexmark Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml) | N/A |`beam_PostCommit_Java_Nexmark_Dataflow_V2.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml?query=event%3Aschedule) |
+| [ PostCommit Java Nexmark Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml) | ['11','17'] |`beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml?query=event%3Aschedule) |
+| [ PostCommit Java Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml) | N/A |`beam_PostCommit_Java_Nexmark_Direct.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit Java Nexmark Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml) | N/A |`beam_PostCommit_Java_Nexmark_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Java Nexmark Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml) | N/A |`beam_PostCommit_Java_Nexmark_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Nexmark_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Java PVR Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml) | N/A |`beam_PostCommit_Java_PVR_Flink_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml?query=event%3Aschedule) |
+| [ PostCommit Java PVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml) | N/A |`beam_PostCommit_Java_PVR_Samza.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Samza.yml?query=event%3Aschedule) |
+| [ PostCommit Java SingleStoreIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml) | N/A |`beam_PostCommit_Java_SingleStoreIO_IT.json`| [![.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml?query=event%3Aschedule) |
+| [ PostCommit Java PVR Spark3 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml) | N/A |`beam_PostCommit_Java_PVR_Spark3_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml?query=event%3Aschedule) |
+| [ PostCommit Java PVR Spark Batch ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml) | N/A |`beam_PostCommit_Java_PVR_Spark_Batch.json`| [![.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml?query=event%3Aschedule) |
+| [ PostCommit Java Sickbay ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml) | N/A |`beam_PostCommit_Java_Sickbay.json`| [![.github/workflows/beam_PostCommit_Java_Sickbay.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Sickbay.yml?query=event%3Aschedule) |
+| [ PostCommit Java Tpcds Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml) | N/A |`beam_PostCommit_Java_Tpcds_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Java Tpcds Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml) | N/A |`beam_PostCommit_Java_Tpcds_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Java Tpcds Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml) | N/A |`beam_PostCommit_Java_Tpcds_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Dataflow JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml) | ['11','17'] |`beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Dataflow V2 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Direct JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml) | ['11','17'] |`beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Direct.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Samza.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner SparkStructuredStreaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner Twister2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Twister2.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml?query=event%3Aschedule) |
+| [ PostCommit Java ValidatesRunner ULR ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_ULR.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml?query=event%3Aschedule) |
+| [ PostCommit Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml) | N/A |`beam_PostCommit_Java.json`| [![.github/workflows/beam_PostCommit_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java.yml?query=event%3Aschedule) |
+| [ PostCommit Javadoc ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml) | N/A |`beam_PostCommit_Javadoc.json`| [![.github/workflows/beam_PostCommit_Javadoc.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Javadoc.yml?query=event%3Aschedule) |
+| [ PostCommit PortableJar Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml) | N/A |`beam_PostCommit_PortableJar_Flink.json`| [![.github/workflows/beam_PostCommit_PortableJar_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit PortableJar Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml) | N/A |`beam_PostCommit_PortableJar_Spark.json`| [![.github/workflows/beam_PostCommit_PortableJar_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_PortableJar_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml) | ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python.json`| [![.github/workflows/beam_PostCommit_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python.yml?query=event%3Aschedule) |
+| [ PostCommit Python Arm](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml) | ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python_Arm.json`| [![.github/workflows/beam_PostCommit_Python_Arm.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Arm.yml?query=event%3Aschedule) |
+| [ PostCommit Python Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml) | N/A |`beam_PostCommit_Python_Examples_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Python Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml) | ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python_Examples_Direct.json`| [![.github/workflows/beam_PostCommit_Python_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit Python Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml) | ['3.8','3.11'] |`beam_PostCommit_Python_Examples_Flink.json`| [![.github/workflows/beam_PostCommit_Python_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Python Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml) | ['3.8','3.11'] |`beam_PostCommit_Python_Examples_Spark.json`| [![.github/workflows/beam_PostCommit_Python_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Examples_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Python MongoDBIO IT ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml) | N/A |`beam_PostCommit_Python_MongoDBIO_IT.json`| [![.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml?query=event%3Aschedule) |
+| [ PostCommit Python Nexmark Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml) | N/A |`beam_PostCommit_Python_Nexmark_Direct.json`| [![.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Nexmark_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit Python ValidatesContainer Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml) | ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python_ValidatesContainer_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Python ValidatesContainer Dataflow With RC ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml) | ['3.8','3.9','3.10','3.11'] |`beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.json`| [![.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml?query=event%3Aschedule) |
+| [ PostCommit Python ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml) | ['3.8','3.11'] |`beam_PostCommit_Python_ValidatesRunner_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Python ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml) | ['3.8','3.11'] |`beam_PostCommit_Python_ValidatesRunner_Flink.json`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit Python ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml) | ['3.8','3.11'] |`beam_PostCommit_Python_ValidatesRunner_Samza.json`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml?query=event%3Aschedule) |
+| [ PostCommit Python ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml) | ['3.8','3.9','3.11'] |`beam_PostCommit_Python_ValidatesRunner_Spark.json`| [![.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml?query=event%3Aschedule) |
+| [ PostCommit Python Xlang Gcp Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml) | N/A |`beam_PostCommit_Python_Xlang_Gcp_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Python Xlang Gcp Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml) | N/A |`beam_PostCommit_Python_Xlang_Gcp_Direct.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit Python Xlang IO Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml) | N/A |`beam_PostCommit_Python_Xlang_IO_Dataflow.json`| [![.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit Sickbay Python ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml) | ['3.8','3.9','3.10','3.11']  |`beam_PostCommit_Sickbay_Python.json`| [![.github/workflows/beam_PostCommit_Sickbay_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Sickbay_Python.yml?query=event%3Aschedule) |
+| [ PostCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml) | N/A |`beam_PostCommit_SQL.json`| [![.github/workflows/beam_PostCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_SQL.yml?query=event%3Aschedule) |
+| [ PostCommit TransformService Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml) | N/A |`beam_PostCommit_TransformService_Direct.json`| [![.github/workflows/beam_PostCommit_TransformService_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_TransformService_Direct.yml?query=event%3Aschedule)
+| [ PostCommit Website Test](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml) | N/A |`beam_PostCommit_Website_Test.json`| [![.github/workflows/beam_PostCommit_Website_Test.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Website_Test.yml?query=event%3Aschedule) |
+| [ PostCommit XVR GoUsingJava Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml) | N/A |`beam_PostCommit_XVR_GoUsingJava_Dataflow.json`| [![.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit XVR Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml) | N/A |`beam_PostCommit_XVR_Direct.json`| [![.github/workflows/beam_PostCommit_XVR_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Direct.yml?query=event%3Aschedule) |
+| [ PostCommit XVR Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml) | N/A |`beam_PostCommit_XVR_Flink.json`| [![.github/workflows/beam_PostCommit_XVR_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Flink.yml?query=event%3Aschedule) |
+| [ PostCommit XVR JavaUsingPython Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml) | N/A |`beam_PostCommit_XVR_JavaUsingPython_Dataflow.json`| [![.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit XVR PythonUsingJava Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml) | N/A |`beam_PostCommit_XVR_PythonUsingJava_Dataflow.json`| [![.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit XVR PythonUsingJavaSQL Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml) | N/A |`beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json`| [![.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml?query=event%3Aschedule) |
+| [ PostCommit XVR Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml) | N/A |`beam_PostCommit_XVR_Samza.json`| [![.github/workflows/beam_PostCommit_XVR_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Samza.yml?query=event%3Aschedule) |
+| [ PostCommit XVR Spark3 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml) | N/A |`beam_PostCommit_XVR_Spark3.json`| [![.github/workflows/beam_PostCommit_XVR_Spark3.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_XVR_Spark3.yml?query=event%3Aschedule) |
+| [ Python Validates Container Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml) | ['3.8','3.9','3.10','3.11'] |`beam_Python_ValidatesContainer_Dataflow_ARM.json`|[![.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml?query=event%3Aschedule) |
 
 ### PerformanceTests and Benchmark Jobs
 
-| Workflow name | Matrix | Trigger Phrase | Cron Status |
-|:-------------:|:------:|:--------------:|:-----------:|
-| [ CloudML Benchmarks Dataflow ](https://github.com/apache/beam/actions/workflows/beam_CloudML_Benchmarks_Dataflow.yml) | N/A |`Run TFT Criteo Benchmarks`| [![.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_CloudML_Benchmarks_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CloudML_Benchmarks_Dataflow.yml?query=event%3Aschedule)
-| [ Inference Python Benchmarks Dataflow ](https://github.com/apache/beam/actions/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml) | N/A |`Run Inference Benchmarks`| [![.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml?query=event%3Aschedule)
-| [ Java JMH ](https://github.com/apache/beam/actions/workflows/beam_Java_JMH.yml) | N/A | N/A | [![.github/workflows/beam_Java_JMH.yml](https://github.com/apache/beam/actions/workflows/beam_Java_JMH.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Java_JMH.yml?query=event%3Aschedule)
-| [ Performance Tests AvroIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml) | N/A |`Run Java AvroIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml?query=event%3Aschedule)
-| [ Performance Tests AvroIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml) | N/A |`Run Java AvroIO Performance Test`| [![.github/workflows/beam_PerformanceTests_AvroIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml?query=event%3Aschedule)
-| [ Performance Tests BigQueryIO Batch Java Avro ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml) | N/A |`Run BigQueryIO Batch Performance Test Java Avro`| [![.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml?query=event%3Aschedule)
-| [ Performance Tests BigQueryIO Batch Java Json ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml) | N/A |`Run BigQueryIO Batch Performance Test Java Json`| [![.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml?query=event%3Aschedule)
-| [ Performance Tests BigQueryIO Streaming Java ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml) | N/A |`Run BigQueryIO Streaming Performance Test Java`| [![.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml?query=event%3Aschedule)
-| [ Performance Tests BigQueryIO Read Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml) | N/A |`Run BigQueryIO Read Performance Test Python`| [![.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml?query=event%3Aschedule)
-| [ Performance Tests BigQueryIO Write Python Batch ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml) | N/A |`Run BigQueryIO Write Performance Test Python`| [![.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml?query=event%3Aschedule)
-| [ PerformanceTests Cdap ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Cdap.yml) | N/A |`Run Java CdapIO Performance Test`| [![.github/workflows/beam_PerformanceTests_Cdap.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Cdap.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Cdap.yml?query=event%3Aschedule)
-| [ PerformanceTests Compressed TextIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml) | N/A |`Run Java CompressedTextIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml?query=event%3Aschedule)
-| [ PerformanceTests Compressed TextIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml) | N/A |`Run Java CompressedTextIO Performance Test`| [![.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml?query=event%3Aschedule)
-| [ PerformanceTests HadoopFormat ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_HadoopFormat.yml) | N/A |`Run Java HadoopFormatIO Performance Test`| [![.github/workflows/beam_PerformanceTests_HadoopFormat.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_HadoopFormat.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_HadoopFormat.yml?query=event%3Aschedule)
-| [ PerformanceTests JDBC ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_JDBC.yml) | N/A |`Run Java JdbcIO Performance Test`| [![.github/workflows/beam_PerformanceTests_JDBC.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_JDBC.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_JDBC.yml?query=event%3Aschedule)
-| [ PerformanceTests Kafka IO ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Kafka_IO.yml) | N/A |`Run Java KafkaIO Performance Test`| [![.github/workflows/beam_PerformanceTests_Kafka_IO.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Kafka_IO.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Kafka_IO.yml?query=event%3Aschedule)
-| [ PerformanceTests ManyFiles TextIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml) | N/A |`Run Java ManyFilesTextIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml?query=event%3Aschedule)
-| [ PerformanceTests ManyFiles TextIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml) | N/A |`Run Java ManyFilesTextIO Performance Test`| [![.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml?query=event%3Aschedule)
-| [ PerformanceTests MongoDBIO IT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_MongoDBIO_IT.yml) | N/A |`Run Java MongoDBIO Performance Test`| [![.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_MongoDBIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_MongoDBIO_IT.yml?query=event%3Aschedule)
-| [ PerformanceTests ParquetIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml) | N/A |`Run Java ParquetIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml?query=event%3Aschedule)
-| [ PerformanceTests ParquetIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT.yml) | N/A |`Run Java ParquetIO Performance Test`| [![.github/workflows/beam_PerformanceTests_ParquetIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT.yml?query=event%3Aschedule)
-| [ PerformanceTests PubsubIOIT Python Streaming ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml) | N/A |`Run PubsubIO Performance Test Python`| [![.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml?query=event%3Aschedule)
-| [ PerformanceTests SingleStoreIO ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SingleStoreIO.yml) | N/A |`Run Java SingleStoreIO Performance Test`| [![.github/workflows/beam_PerformanceTests_SingleStoreIO.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SingleStoreIO.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SingleStoreIO.yml?query=event%3Aschedule)
-| [ PerformanceTests SpannerIO Read 2GB Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml) | N/A |`Run SpannerIO Read 2GB Performance Test Python`| [![.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml?query=event%3Aschedule)
-| [ PerformanceTests SpannerIO Write 2GB Python Batch ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml) | N/A |`Run SpannerIO Write 2GB Performance Test Python Batch`| [![.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml?query=event%3Aschedule)
-| [ PerformanceTests SparkReceiver IO ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SparkReceiver_IO.yml) | N/A |`Run Java SparkReceiverIO Performance Test`| [![.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SparkReceiver_IO.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SparkReceiver_IO.yml?query=event%3Aschedule)
-| [ PerformanceTests SQLBigQueryIO Batch Java ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml) | N/A |`Run SQLBigQueryIO Batch Performance Test Java`| [![.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml?query=event%3Aschedule)
-| [ PerformanceTests TextIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml) | N/A |`Run Java TextIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml?query=event%3Aschedule)
-| [ PerformanceTests TextIOIT Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_Python.yml) | N/A |`Run Python TextIO Performance Test`| [![.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_Python.yml?query=event%3Aschedule)
-| [ PerformanceTests TextIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT.yml) | N/A |`Run Java TextIO Performance Test`| [![.github/workflows/beam_PerformanceTests_TextIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT.yml?query=event%3Aschedule)
-| [ PerformanceTests TFRecordIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml) | N/A |`Run Java TFRecordIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml?query=event%3Aschedule)
-| [ PerformanceTests TFRecordIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT.yml) | N/A |`Run Java TFRecordIO Performance Test`| [![.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT.yml?query=event%3Aschedule)
-| [ PerformanceTests WordCountIT PythonVersions ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml) | ['3.8'] |`Run Python (matrix_element) WordCountIT Performance Test`| [![.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml?query=event%3Aschedule)
-| [ PerformanceTests XmlIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml) | N/A |`Run Java XmlIO Performance Test HDFS`| [![.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml?query=event%3Aschedule)
-| [ PerformanceTests XmlIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT.yml) | N/A |`Run Java XmlIO Performance Test`| [![.github/workflows/beam_PerformanceTests_XmlIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT.yml?query=event%3Aschedule)
-| [ PerformanceTests xlang KafkaIO Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml) | N/A |`Run Python xlang KafkaIO Performance Test`| [![.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml?query=event%3Aschedule)
+| Workflow name | Matrix | Cron Status |
+|:-------------:|:------:|:-----------:|
+| [ CloudML Benchmarks Dataflow ](https://github.com/apache/beam/actions/workflows/beam_CloudML_Benchmarks_Dataflow.yml) | N/A | [![.github/workflows/beam_CloudML_Benchmarks_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_CloudML_Benchmarks_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CloudML_Benchmarks_Dataflow.yml?query=event%3Aschedule)
+| [ Inference Python Benchmarks Dataflow ](https://github.com/apache/beam/actions/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml) | N/A | [![.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml?query=event%3Aschedule)
+| [ Java JMH ](https://github.com/apache/beam/actions/workflows/beam_Java_JMH.yml) | N/A | [![.github/workflows/beam_Java_JMH.yml](https://github.com/apache/beam/actions/workflows/beam_Java_JMH.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Java_JMH.yml?query=event%3Aschedule)
+| [ Performance Tests AvroIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT_HDFS.yml?query=event%3Aschedule)
+| [ Performance Tests AvroIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_AvroIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_AvroIOIT.yml?query=event%3Aschedule)
+| [ Performance Tests BigQueryIO Batch Java Avro ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml) | N/A | [![.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml?query=event%3Aschedule)
+| [ Performance Tests BigQueryIO Batch Java Json ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml) | N/A | [![.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml?query=event%3Aschedule)
+| [ Performance Tests BigQueryIO Streaming Java ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml) | N/A | [![.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml?query=event%3Aschedule)
+| [ Performance Tests BigQueryIO Read Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml) | N/A | [![.github/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Read_Python.yml?query=event%3Aschedule)
+| [ Performance Tests BigQueryIO Write Python Batch ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml) | N/A | [![.github/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_BiqQueryIO_Write_Python_Batch.yml?query=event%3Aschedule)
+| [ PerformanceTests Cdap ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Cdap.yml) | N/A | [![.github/workflows/beam_PerformanceTests_Cdap.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Cdap.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Cdap.yml?query=event%3Aschedule)
+| [ PerformanceTests Compressed TextIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT_HDFS.yml?query=event%3Aschedule)
+| [ PerformanceTests Compressed TextIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Compressed_TextIOIT.yml?query=event%3Aschedule)
+| [ PerformanceTests HadoopFormat ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_HadoopFormat.yml) | N/A | [![.github/workflows/beam_PerformanceTests_HadoopFormat.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_HadoopFormat.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_HadoopFormat.yml?query=event%3Aschedule)
+| [ PerformanceTests JDBC ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_JDBC.yml) | N/A | [![.github/workflows/beam_PerformanceTests_JDBC.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_JDBC.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_JDBC.yml?query=event%3Aschedule)
+| [ PerformanceTests Kafka IO ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Kafka_IO.yml) | N/A | [![.github/workflows/beam_PerformanceTests_Kafka_IO.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Kafka_IO.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_Kafka_IO.yml?query=event%3Aschedule)
+| [ PerformanceTests ManyFiles TextIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT_HDFS.yml?query=event%3Aschedule)
+| [ PerformanceTests ManyFiles TextIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ManyFiles_TextIOIT.yml?query=event%3Aschedule)
+| [ PerformanceTests MongoDBIO IT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_MongoDBIO_IT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_MongoDBIO_IT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_MongoDBIO_IT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_MongoDBIO_IT.yml?query=event%3Aschedule)
+| [ PerformanceTests ParquetIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT_HDFS.yml?query=event%3Aschedule)
+| [ PerformanceTests ParquetIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_ParquetIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_ParquetIOIT.yml?query=event%3Aschedule)
+| [ PerformanceTests PubsubIOIT Python Streaming ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml) | N/A | [![.github/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_PubsubIOIT_Python_Streaming.yml?query=event%3Aschedule)
+| [ PerformanceTests SingleStoreIO ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SingleStoreIO.yml) | N/A | [![.github/workflows/beam_PerformanceTests_SingleStoreIO.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SingleStoreIO.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SingleStoreIO.yml?query=event%3Aschedule)
+| [ PerformanceTests SpannerIO Read 2GB Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml) | N/A | [![.github/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Read_2GB_Python.yml?query=event%3Aschedule)
+| [ PerformanceTests SpannerIO Write 2GB Python Batch ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml) | N/A | [![.github/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SpannerIO_Write_2GB_Python_Batch.yml?query=event%3Aschedule)
+| [ PerformanceTests SparkReceiver IO ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SparkReceiver_IO.yml) | N/A | [![.github/workflows/beam_PerformanceTests_SparkReceiver_IO.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SparkReceiver_IO.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SparkReceiver_IO.yml?query=event%3Aschedule)
+| [ PerformanceTests SQLBigQueryIO Batch Java ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml) | N/A | [![.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml?query=event%3Aschedule)
+| [ PerformanceTests TextIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_HDFS.yml?query=event%3Aschedule)
+| [ PerformanceTests TextIOIT Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_Python.yml) | N/A | [![.github/workflows/beam_PerformanceTests_TextIOIT_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT_Python.yml?query=event%3Aschedule)
+| [ PerformanceTests TextIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_TextIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TextIOIT.yml?query=event%3Aschedule)
+| [ PerformanceTests TFRecordIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT_HDFS.yml?query=event%3Aschedule)
+| [ PerformanceTests TFRecordIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_TFRecordIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_TFRecordIOIT.yml?query=event%3Aschedule)
+| [ PerformanceTests WordCountIT PythonVersions ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml) | ['3.8'] | [![.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml?query=event%3Aschedule)
+| [ PerformanceTests XmlIOIT HDFS ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml) | N/A | [![.github/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT_HDFS.yml?query=event%3Aschedule)
+| [ PerformanceTests XmlIOIT ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT.yml) | N/A | [![.github/workflows/beam_PerformanceTests_XmlIOIT.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_XmlIOIT.yml?query=event%3Aschedule)
+| [ PerformanceTests xlang KafkaIO Python ](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml) | N/A | [![.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml?query=event%3Aschedule)
 
 ### LoadTests Jobs
 
-| Workflow name | Matrix | Trigger Phrase | Cron Status |
-|:-------------:|:------:|:--------------:|:-----------:|
-| [ LoadTests Go CoGBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml) | N/A |`Run LoadTests Go CoGBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go CoGBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml) | N/A |`Run Load Tests Go CoGBK Flink Batch`| [![.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml?query=event%3Aschedule)
-| [ LoadTests Go Combine Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml) | N/A |`Run Load Tests Go Combine Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go Combine Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml) | N/A |`Run Load Tests Go Combine Flink Batch`| [![.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml) | N/A |`Run Load Tests Go GBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go GBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml) | N/A |`Run Load Tests Go GBK Flink Batch`| [![.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go ParDo Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml) | N/A |`Run Load Tests Go ParDo Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go ParDo Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml) | N/A |`Run Load Tests Go ParDo Flink Batch`| [![.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go SideInput Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml) | N/A |`Run Load Tests Go SideInput Dataflow Batch`| [![.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Go SideInput Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml) | N/A |`Run Load Tests Go SideInput Flink Batch`| [![.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java CoGBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml) | N/A |`Run Load Tests Java CoGBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java CoGBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml) | N/A |`Run Load Tests Java CoGBK Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Java CoGBK Dataflow V2 Batch JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml) | ['11','17'] |`Run Load Tests Java (matrix_element) CoGBK Dataflow V2 Batch`| [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml?query=event%3Aschedule)
-| [ LoadTests Java CoGBK Dataflow V2 Streaming JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml) | ['11','17'] |`Run Load Tests Java (matrix_element) CoGBK Dataflow V2 Streaming`| [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml?query=event%3Aschedule)
-| [ LoadTests Java CoGBK SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml) | N/A |`Run Load Tests Java CoGBK SparkStructuredStreaming Batch`| [![.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java Combine Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml) | N/A |`Run Load Tests Java Combine Dataflow Batch`| [![.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java Combine Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml) | N/A |`Run Load Tests Java Combine Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Java Combine SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml) | N/A |`Run Load Tests Java Combine SparkStructuredStreaming Batch`| [![.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml) | N/A |`Run Load Tests Java GBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml) | N/A |`Run Load Tests Java GBK Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Dataflow V2 Batch Java11 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml) | N/A |`Run Load Tests Java 11 GBK Dataflow V2 Batch`| [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Dataflow V2 Batch Java17 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml) | N/A |`Run Load Tests Java 17 GBK Dataflow V2 Batch`| [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Dataflow V2 Streaming Java11 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml) | N/A |`Run Load Tests Java 11 GBK Dataflow V2 Streaming`| [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Dataflow V2 Streaming Java17 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml) | N/A |`Run Load Tests Java 17 GBK Dataflow V2 Streaming`| [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK Smoke ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml) | N/A |`Run Java Load Tests GBK Smoke`| [![.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml?query=event%3Aschedule)
-| [ LoadTests Java GBK SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml) | N/A |`Run Load Tests Java GBK SparkStructuredStreaming Batch`| [![.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java ParDo Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml) | N/A |`Run Load Tests Java ParDo Dataflow Batch`| [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java ParDo Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml) | N/A |`Run Load Tests Java ParDo Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Java ParDo Dataflow V2 Batch JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml) | ['11','17'] |`Run Load Tests Java (matrix_element) ParDo Dataflow V2 Batch`| [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml?query=event%3Aschedule)
-| [ LoadTests Java ParDo Dataflow V2 Streaming JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml) | ['11','17'] |`Run Load Tests Java (matrix_element) ParDo Dataflow V2 Streaming`| [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml?query=event%3Aschedule)
-| [ LoadTests Java ParDo SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml) | N/A |`Run Load Tests Java ParDo SparkStructuredStreaming Batch`| [![.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Java Combine Smoke ](https://github.com/apache/beam/actions/workflows/beam_Java_LoadTests_Combine_Smoke.yml) | N/A | N/A | [![.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_Java_LoadTests_Combine_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Java_LoadTests_Combine_Smoke.yml?query=event%3Aschedule)
-| [ LoadTests Python CoGBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml) | N/A |`Run Load Tests Python CoGBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python CoGBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml) | N/A |`Run Load Tests Python CoGBK Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python CoGBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml) | N/A |`Run Load Tests Python CoGBK Flink Batch`| [![.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python Combine Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml) | N/A |`Run Load Tests Python Combine Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python Combine Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml) | N/A |`Run Load Tests Python Combine Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python Combine Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml) | N/A |`Run Load Tests Python Combine Flink Batch`| [![.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python Combine Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml) | N/A |`Run Load Tests Python Combine Flink Streaming`| [![.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python FnApiRunner Microbenchmark ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml) | N/A |`Run Python Load Tests FnApiRunner Microbenchmark`| [![.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml?query=event%3Aschedule)
-| [ LoadTests Python GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml) | N/A |`Run Load Tests Python GBK Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python GBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml) | N/A |`Run Load Tests Python GBK Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python GBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml) | N/A |`Run Load Tests Python GBK Flink Batch`| [![.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python GBK reiterate Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml) | N/A |`Run Load Tests Python GBK reiterate Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python GBK reiterate Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml) | N/A |`Run Load Tests Python GBK reiterate Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python ParDo Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml) | N/A |`Run Load Tests Python ParDo Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python ParDo Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml) | N/A |`Run Python Load Tests ParDo Dataflow Streaming`| [![.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python ParDo Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml) | N/A |`Run Load Tests Python ParDo Flink Batch`| [![.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python ParDo Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml) | N/A |`Run Load Tests Python ParDo Flink Streaming`| [![.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml?query=event%3Aschedule)
-| [ LoadTests Python SideInput Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml) | N/A |`Run Load Tests Python SideInput Dataflow Batch`| [![.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml?query=event%3Aschedule)
-| [ LoadTests Python Smoke ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Smoke.yml) | N/A |`Run Python Load Tests Smoke`| [![.github/workflows/beam_LoadTests_Python_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Smoke.yml?query=event%3Aschedule)
+| Workflow name | Matrix | Cron Status |
+|:-------------:|:------:|:-----------:|
+| [ LoadTests Go CoGBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go CoGBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_CoGBK_Flink_batch.yml?query=event%3Aschedule)
+| [ LoadTests Go Combine Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go Combine Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_Combine_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go GBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_GBK_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go ParDo Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go ParDo Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_ParDo_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go SideInput Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Go SideInput Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Go_SideInput_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java CoGBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java CoGBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Java CoGBK Dataflow V2 Batch JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml) | ['11','17'] | [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml?query=event%3Aschedule)
+| [ LoadTests Java CoGBK Dataflow V2 Streaming JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml) | ['11','17'] | [![.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml?query=event%3Aschedule)
+| [ LoadTests Java CoGBK SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_CoGBK_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java Combine Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java Combine Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Java Combine SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Dataflow V2 Batch Java11 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Dataflow V2 Batch Java17 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Dataflow V2 Streaming Java11 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Dataflow V2 Streaming Java17 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK Smoke ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml?query=event%3Aschedule)
+| [ LoadTests Java GBK SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java ParDo Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java ParDo Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Java ParDo Dataflow V2 Batch JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml) | ['11','17'] | [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml?query=event%3Aschedule)
+| [ LoadTests Java ParDo Dataflow V2 Streaming JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml) | ['11','17'] | [![.github/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml?query=event%3Aschedule)
+| [ LoadTests Java ParDo SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_ParDo_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Java Combine Smoke ](https://github.com/apache/beam/actions/workflows/beam_Java_LoadTests_Combine_Smoke.yml) | N/A | [![.github/workflows/beam_Java_LoadTests_Combine_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_Java_LoadTests_Combine_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Java_LoadTests_Combine_Smoke.yml?query=event%3Aschedule)
+| [ LoadTests Python CoGBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python CoGBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python CoGBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_CoGBK_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python Combine Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python Combine Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python Combine Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python Combine Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Combine_Flink_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python FnApiRunner Microbenchmark ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_FnApiRunner_Microbenchmark.yml?query=event%3Aschedule)
+| [ LoadTests Python GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python GBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python GBK Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python GBK reiterate Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python GBK reiterate Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python ParDo Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python ParDo Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python ParDo Flink Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python ParDo Flink Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml?query=event%3Aschedule)
+| [ LoadTests Python SideInput Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_SideInput_Dataflow_Batch.yml?query=event%3Aschedule)
+| [ LoadTests Python Smoke ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Smoke.yml) | N/A | [![.github/workflows/beam_LoadTests_Python_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Python_Smoke.yml?query=event%3Aschedule)
 
 ### Other Jobs
 
-| Workflow name | Matrix | Trigger Phrase | Cron Status |
-|:-------------:|:------:|:--------------:|:-----------:|
-| [ Cancel Stale Dataflow Jobs ](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml) | N/A | `Run Cancel Stale Dataflow Jobs` | [![.github/workflows/beam_CancelStaleDataflowJobs.yml](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml?query=event%3Aschedule) |
-| [ Clean Up GCP Resources ](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml) | N/A | `Run Clean GCP Resources` | [![.github/workflows/beam_CleanUpGCPResources.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml?query=event%3Aschedule) |
-| [ Clean Up Prebuilt SDK Images ](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml) | N/A | `Run Clean Prebuilt Images` | [![.github/workflows/beam_beam_CleanUpPrebuiltSDKImages.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml?query=event%3Aschedule) |
-| [ Cleanup Dataproc Resources ](https://github.com/apache/beam/actions/workflows/beam_CleanUpDataprocResources.yml) | N/A | N/A | [![.github/workflows/beam_CleanUpDataprocResources.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpDataprocResources.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpDataprocResources.yml?query=event%3Aschedule)
-| [ Community Metrics Prober ](https://github.com/apache/beam/actions/workflows/beam_Prober_CommunityMetrics.yml) | N/A |`Run Community Metrics Prober`| [![.github/workflows/beam_Prober_CommunityMetrics.yml](https://github.com/apache/beam/actions/workflows/beam_Prober_CommunityMetrics.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Prober_CommunityMetrics.yml?query=event%3Aschedule)
-| [ Publish Beam SDK Snapshots ](https://github.com/apache/beam/actions/workflows/beam_Publish_Beam_SDK_Snapshots.yml) | N/A | N/A | [![.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml](https://github.com/apache/beam/actions/workflows/beam_Publish_Beam_SDK_Snapshots.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Publish_Beam_SDK_Snapshots.yml?query=event%3Aschedule) |
-| [ Publish Docker Snapshots ](https://github.com/apache/beam/actions/workflows/beam_Publish_Docker_Snapshots.yml) | N/A |`Publish Docker Snapshots`| [![.github/workflows/beam_Publish_Docker_Snapshots.yml](https://github.com/apache/beam/actions/workflows/beam_Publish_Docker_Snapshots.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Publish_Docker_Snapshots.yml?query=event%3Aschedule) |
-| [ Rotate IO-Datastores Cluster Credentials ](https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml) | N/A | N/A | [![.github/workflows/beam_IODatastoresCredentialsRotation.yml](https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml?query=event%3Aschedule) |
-| [ Rotate Metrics Cluster Credentials ](https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml) | N/A | N/A | [![.github/workflows/beam_MetricsCredentialsRotation.yml](https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml?query=event%3Aschedule) |
+| Workflow name | Matrix | Cron Status |
+|:-------------:|:------:|:-----------:|
+| [ Publish BeamMetrics ](https://github.com/apache/beam/actions/workflows/beam_Publish_BeamMetrics.yml) | N/A | [![.github/workflows/beam_Publish_BeamMetrics.yml](https://github.com/apache/beam/actions/workflows/beam_Publish_BeamMetrics.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Publish_BeamMetrics.yml?query=event%3Aschedule)
+| [ Publish Website  ](https://github.com/apache/beam/actions/workflows/beam_Publish_Website.yml) | N/A | [![.github/workflows/beam_Publish_Website.yml](https://github.com/apache/beam/actions/workflows/beam_Publish_Website.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Publish_Website.yml?query=event%3Aschedule) |
+| [ Cancel Stale Dataflow Jobs ](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml) | N/A | [![.github/workflows/beam_CancelStaleDataflowJobs.yml](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CancelStaleDataflowJobs.yml?query=event%3Aschedule) |
+| [ Clean Up GCP Resources ](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml) | N/A | [![.github/workflows/beam_CleanUpGCPResources.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpGCPResources.yml?query=event%3Aschedule) |
+| [ Clean Up Prebuilt SDK Images ](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml) | N/A | [![.github/workflows/beam_beam_CleanUpPrebuiltSDKImages.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpPrebuiltSDKImages.yml?query=event%3Aschedule) |
+| [ Cleanup Dataproc Resources ](https://github.com/apache/beam/actions/workflows/beam_CleanUpDataprocResources.yml) | N/A | [![.github/workflows/beam_CleanUpDataprocResources.yml](https://github.com/apache/beam/actions/workflows/beam_CleanUpDataprocResources.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_CleanUpDataprocResources.yml?query=event%3Aschedule)
+| [ Community Metrics Prober ](https://github.com/apache/beam/actions/workflows/beam_Prober_CommunityMetrics.yml) | N/A | [![.github/workflows/beam_Prober_CommunityMetrics.yml](https://github.com/apache/beam/actions/workflows/beam_Prober_CommunityMetrics.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Prober_CommunityMetrics.yml?query=event%3Aschedule)
+| [ Publish Beam SDK Snapshots ](https://github.com/apache/beam/actions/workflows/beam_Publish_Beam_SDK_Snapshots.yml) | N/A | [![.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml](https://github.com/apache/beam/actions/workflows/beam_Publish_Beam_SDK_Snapshots.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Publish_Beam_SDK_Snapshots.yml?query=event%3Aschedule) |
+| [ Publish Docker Snapshots ](https://github.com/apache/beam/actions/workflows/beam_Publish_Docker_Snapshots.yml) | N/A | [![.github/workflows/beam_Publish_Docker_Snapshots.yml](https://github.com/apache/beam/actions/workflows/beam_Publish_Docker_Snapshots.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_Publish_Docker_Snapshots.yml?query=event%3Aschedule) |
+| [ Rotate IO-Datastores Cluster Credentials ](https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml) | N/A | [![.github/workflows/beam_IODatastoresCredentialsRotation.yml](https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_IODatastoresCredentialsRotation.yml?query=event%3Aschedule) |
+| [ Rotate Metrics Cluster Credentials ](https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml) | N/A | [![.github/workflows/beam_MetricsCredentialsRotation.yml](https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_MetricsCredentialsRotation.yml?query=event%3Aschedule) |
diff --git a/.github/workflows/beam_Java_JMH.yml b/.github/workflows/beam_Java_JMH.yml
index ccd3162873ad..7764e7223472 100644
--- a/.github/workflows/beam_Java_JMH.yml
+++ b/.github/workflows/beam_Java_JMH.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 0 * * 0'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_Java_JMH.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -49,6 +49,8 @@ env:
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
   INFLUXDB_HOST: http://10.128.0.96:8086
   INFLUXDB_DATABASE: beam_test_metrics
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_Java_JMH:
diff --git a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml
index ffb38e34a454..881d5dedaac5 100644
--- a/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml
+++ b/.github/workflows/beam_LoadTests_Java_CoGBK_Dataflow_Streaming.yml
@@ -115,8 +115,8 @@ jobs:
             -Prunner=:runners:google-cloud-dataflow-java \
             '-PloadTest.args=${{ env.beam_LoadTests_Java_CoGBK_Dataflow_Streaming_test_arguments_4 }}' \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_Metrics_Report.yml b/.github/workflows/beam_Metrics_Report.yml
index b07e64b12cfc..3ed5850e5dba 100644
--- a/.github/workflows/beam_Metrics_Report.yml
+++ b/.github/workflows/beam_Metrics_Report.yml
@@ -74,7 +74,7 @@ jobs:
           INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
           INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
       - name: Archive Report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Metrics Report
           path: "${{ github.workspace }}/.test-infra/jenkins/metrics_report/beam-metrics_report.html"
diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml
index 8727b2387403..8b9eefcd8052 100644
--- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml
+++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Avro.yml
@@ -91,8 +91,8 @@ jobs:
             -DintegrationTestRunner=dataflow \
             -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Batch_Java_Avro_test_arguments_1 }}]' \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml
index a231d00f5ede..1fb81b9b1431 100644
--- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml
+++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Batch_Java_Json.yml
@@ -91,8 +91,8 @@ jobs:
             -DintegrationTestRunner=dataflow \
             -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Batch_Java_Json_test_arguments_1 }}]' \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml b/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml
index 7259fb1838f2..c0433043f1ed 100644
--- a/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml
+++ b/.github/workflows/beam_PerformanceTests_BigQueryIO_Streaming_Java.yml
@@ -91,8 +91,8 @@ jobs:
             -DintegrationTestRunner=dataflow \
             -DintegrationTestPipelineOptions='[${{ env.beam_PerformanceTests_BigQueryIO_Streaming_Java_test_arguments_1 }}]' \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml b/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml
index c02d16688a69..dbe495f12cd1 100644
--- a/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml
+++ b/.github/workflows/beam_PerformanceTests_SQLBigQueryIO_Batch_Java.yml
@@ -90,8 +90,8 @@ jobs:
             -DintegrationTestRunner=dataflow  \
             '-DintegrationTestPipelineOptions=[${{env.beam_PerformanceTests_SQLBigQueryIO_Batch_Java_test_arguments_1}}]' \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml
index b3e606fcc390..b814e5d99924 100644
--- a/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml
+++ b/.github/workflows/beam_PerformanceTests_WordCountIT_PythonVersions.yml
@@ -104,7 +104,7 @@ jobs:
             -Ptest=apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it \
             "-Ptest-pipeline-options=${{ env.beam_PerformanceTests_WordCountIT_PythonVersions_test_arguments_1 }}"
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Go.yml b/.github/workflows/beam_PostCommit_Go.yml
index f9da27ff2ce9..821ec55e2d1b 100644
--- a/.github/workflows/beam_PostCommit_Go.yml
+++ b/.github/workflows/beam_PostCommit_Go.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Go.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
diff --git a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml
index 532c48e7b072..615b7f1f94b2 100644
--- a/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml
+++ b/.github/workflows/beam_PostCommit_Go_Dataflow_ARM.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Go_Dataflow_ARM.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
diff --git a/.github/workflows/beam_PostCommit_Go_VR_Flink.yml b/.github/workflows/beam_PostCommit_Go_VR_Flink.yml
index ce1e82d22e2a..c85f0c00035a 100644
--- a/.github/workflows/beam_PostCommit_Go_VR_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Go_VR_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Go_VR_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
diff --git a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
index 5d1ab6a8d13e..3ac3742500fc 100644
--- a/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
+++ b/.github/workflows/beam_PostCommit_Go_VR_Samza.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Go_VR_Samza.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
diff --git a/.github/workflows/beam_PostCommit_Go_VR_Spark.yml b/.github/workflows/beam_PostCommit_Go_VR_Spark.yml
index ed9f50583133..d85038befe38 100644
--- a/.github/workflows/beam_PostCommit_Go_VR_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Go_VR_Spark.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Go_VR_Spark.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
diff --git a/.github/workflows/beam_PostCommit_Java.yml b/.github/workflows/beam_PostCommit_Java.yml
index d9fa4d52f965..cc7d1c7d64ad 100644
--- a/.github/workflows/beam_PostCommit_Java.yml
+++ b/.github/workflows/beam_PostCommit_Java.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :javaPostCommit
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml b/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml
index 36bdec265dd5..313f5c1340a2 100644
--- a/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml
+++ b/.github/workflows/beam_PostCommit_Java_Avro_Versions.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
             gradle-command: :javaAvroVersionsTest
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml b/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml
index f21f301cc265..f832c5fab562 100644
--- a/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml
+++ b/.github/workflows/beam_PostCommit_Java_BigQueryEarlyRollout.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_BigQueryEarlyRollout.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -86,8 +86,8 @@ jobs:
         with:
           gradle-command: :sdks:java:io:google-cloud-platform:bigQueryEarlyRolloutIntegrationTest
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_DataflowV1.yml b/.github/workflows/beam_PostCommit_Java_DataflowV1.yml
index 68c901c2e527..6e31094fa468 100644
--- a/.github/workflows/beam_PostCommit_Java_DataflowV1.yml
+++ b/.github/workflows/beam_PostCommit_Java_DataflowV1.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_DataflowV1.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -83,8 +83,8 @@ jobs:
         with:
           gradle-command: :runners:google-cloud-dataflow-java:postCommit
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_DataflowV2.yml b/.github/workflows/beam_PostCommit_Java_DataflowV2.yml
index 90a97296df77..58ecc37361d4 100644
--- a/.github/workflows/beam_PostCommit_Java_DataflowV2.yml
+++ b/.github/workflows/beam_PostCommit_Java_DataflowV2.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_DataflowV2.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :runners:google-cloud-dataflow-java:postCommitRunnerV2
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml
index 488031a9244a..183a55a8e742 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml
@@ -21,7 +21,7 @@ on:
   schedule:
   - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -80,8 +80,8 @@ jobs:
         gradle-command: :runners:google-cloud-dataflow-java:examples:javaPostCommit
         max-workers: 12
     - name: Archive JUnit Test Results
-      uses: actions/upload-artifact@v3
-      if: failure()
+      uses: actions/upload-artifact@v4
+      if: ${{ !success() }}
       with:
         name: JUnit Test Results
         path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml
index 26ea487e0dcf..33557dc6d2d1 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml
@@ -23,7 +23,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_ARM.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -113,8 +113,8 @@ jobs:
                 -PdisableCheckStyle=true \
                 -PskipCheckerFramework \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml
index 747c9f0983fc..f9e828106bf5 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -86,8 +86,8 @@ jobs:
           gradle-command: :runners:google-cloud-dataflow-java:examples:java${{ matrix.java_version }}PostCommit
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml
index ae50ec506dff..661dd2dde2b1 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_V2.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -80,8 +80,8 @@ jobs:
             gradle-command: :runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest
             max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml
index 1ed3cf5eb981..18fa9513ac63 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_V2_Java.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -94,8 +94,8 @@ jobs:
             -PtestJavaVersion=${{ matrix.java_version }} \
             -Pjava${{ matrix.java_version }}Home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml
index fa7e76942808..34d17b4b20a9 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Direct.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Direct.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -81,8 +81,8 @@ jobs:
         with:
           gradle-command: :runners:direct:examplesIntegrationTest
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml
index 8e7a99f12ac8..3131422fe732 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Flink.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -82,8 +82,8 @@ jobs:
         with:
           gradle-command: :runners:flink:1.15:examplesIntegrationTest
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml
index f927fa8b3b46..4bc8408f9a4d 100644
--- a/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Examples_Spark.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -81,8 +81,8 @@ jobs:
         with:
           gradle-command: :runners:spark:3:examplesIntegrationTest
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml
index c0c85fb0cd31..a227144e6570 100644
--- a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml
+++ b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 3/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Hadoop_Versions.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :javaHadoopVersionsTest
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
index 4bbc23b13de4..588da4ea61a2 100644
--- a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
+++ b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml
@@ -21,7 +21,7 @@ on:
   schedule:
   - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_IO_Performance_Tests.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -107,8 +107,8 @@ jobs:
         exportDataset: performance_tests
         exportTable: io_performance_metrics_test
     - name: Archive JUnit Test Results
-      uses: actions/upload-artifact@v3
-      if: failure()
+      uses: actions/upload-artifact@v4
+      if: ${{ !success() }}
       with:
         name: JUnit Test Results
         path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml b/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml
index c1f75be3851a..0bfbdedadcca 100644
--- a/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml
+++ b/.github/workflows/beam_PostCommit_Java_InfluxDbIO_IT.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_InfluxDbIO_IT.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml
index 2449054a1073..0f39e0d02537 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java11.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Dataflow_Java11.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -80,8 +80,8 @@ jobs:
           gradle-command: :sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest
           arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml
index 611c8a9d31de..049b5e27e341 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Dataflow_Java17.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Dataflow_Java17.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -85,8 +85,8 @@ jobs:
             -PtestJavaVersion=17
             -Pjava17Home=$JAVA_HOME_17_X64
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml
index 6fdff9c451eb..73750453e5e3 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java11.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Direct_Java11.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -80,8 +80,8 @@ jobs:
           gradle-command: :sdks:java:testing:jpms-tests:directRunnerIntegrationTest
           arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml
index 3f62861eb025..2a56f3e96201 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java17.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Direct_Java17.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -85,8 +85,8 @@ jobs:
             -PtestJavaVersion=17
             -Pjava17Home=$JAVA_HOME_17_X64
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml
index e1926a430069..163169563047 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Direct_Java21.yml
@@ -19,7 +19,7 @@ on:
   schedule:
   - cron: '0 */6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Direct_Java21.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -71,7 +71,7 @@ jobs:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
     - name: Set up Java
-      uses: actions/setup-java@v3.11.0
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: |
@@ -86,8 +86,8 @@ jobs:
           -PtestJavaVersion=21
           -Pjava21Home=$JAVA_HOME_21_X64
     - name: Archive JUnit Test Results
-      uses: actions/upload-artifact@v3
-      if: failure()
+      uses: actions/upload-artifact@v4
+      if: ${{ !success() }}
       with:
         name: JUnit Test Results
         path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml
index fbc0d16e4994..ea8a89def8ee 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Flink_Java11.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Flink_Java11.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -80,8 +80,8 @@ jobs:
           gradle-command: :sdks:java:testing:jpms-tests:flinkRunnerIntegrationTest
           arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml b/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml
index 2c2e2acc6f19..a8754ac254d8 100644
--- a/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml
+++ b/.github/workflows/beam_PostCommit_Java_Jpms_Spark_Java11.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Jpms_Spark_Java11.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -80,8 +80,8 @@ jobs:
           gradle-command: :sdks:java:testing:jpms-tests:sparkRunnerIntegrationTest
           arguments: -Dorg.gradle.java.home=$JAVA_HOME_11_X64
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml
index f4e96961061e..ab4c63e772ea 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Nexmark_Dataflow.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -72,6 +72,8 @@ env:
     --enforceEncodability=true 
     --enforceImmutability=true 
     --runner=DataflowRunner
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Nexmark_Dataflow:
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml
index 875f54ea7632..63321d8aa6c1 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Nexmark_Dataflow_V2.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -73,6 +73,8 @@ env:
     --enforceEncodability=true 
     --enforceImmutability=true 
     --runner=DataflowRunner
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Nexmark_Dataflow_V2:
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml
index ba3f8bb61005..1a4c38c46914 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '0 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Nexmark_Dataflow_V2_Java.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -72,6 +72,8 @@ env:
     --enforceEncodability=true 
     --enforceImmutability=true 
     --runner=DataflowRunner
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Nexmark_Dataflow_V2_Java:
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml
index 23d766c89823..5e1975b9ae83 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Direct.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Nexmark_Direct.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -67,6 +67,8 @@ env:
     --enforceEncodability=true 
     --enforceImmutability=true 
     --runner=DirectRunner
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Nexmark_Direct:
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
index dd87fec145a4..405a6399d2e8 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Flink.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Nexmark_Flink.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -66,6 +66,8 @@ env:
     --suite=SMOKE 
     --streamTimeout=60 
     --runner=FlinkRunner
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Nexmark_Flink:
diff --git a/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml b/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml
index 05229fb4c89b..29291f441ce0 100644
--- a/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Java_Nexmark_Spark.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Nexmark_Spark.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -66,6 +66,8 @@ env:
     --suite=SMOKE 
     --streamTimeout=60 
     --streaming=false
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
 
 jobs:
   beam_PostCommit_Java_Nexmark_Spark:
diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml b/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml
index cffe074f58eb..13d3384f8087 100644
--- a/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml
+++ b/.github/workflows/beam_PostCommit_Java_PVR_Flink_Streaming.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Streaming.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
             gradle-command: runners:flink:1.15:job-server:validatesPortableRunnerStreaming
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml b/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml
index 729e95fe5219..86647b6079d3 100644
--- a/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml
+++ b/.github/workflows/beam_PostCommit_Java_PVR_Samza.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_PVR_Samza.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -81,8 +81,8 @@ jobs:
         with:
             gradle-command: :runners:samza:job-server:validatesPortableRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml b/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml
index d3508e968fe0..58ee88e20ec3 100644
--- a/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml
+++ b/.github/workflows/beam_PostCommit_Java_PVR_Spark3_Streaming.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
             gradle-command: :runners:spark:3:job-server:validatesPortableRunnerStreaming
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml b/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml
index 8cc977ddea82..a6061da45e92 100644
--- a/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml
+++ b/.github/workflows/beam_PostCommit_Java_PVR_Spark_Batch.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -82,8 +82,8 @@ jobs:
               :runners:spark:3:job-server:validatesPortableRunnerBatch \
               :runners:spark:3:job-server:validatesPortableRunnerDocker \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -96,7 +96,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: SpotBugs Results
           path: "**/build/reports/spotbugs/*.html"
diff --git a/.github/workflows/beam_PostCommit_Java_Sickbay.yml b/.github/workflows/beam_PostCommit_Java_Sickbay.yml
index e6de4b2538b5..2e96ae63382c 100644
--- a/.github/workflows/beam_PostCommit_Java_Sickbay.yml
+++ b/.github/workflows/beam_PostCommit_Java_Sickbay.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Sickbay.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
             gradle-command: :javaPostCommitSickbay
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml b/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml
index 2fff2365d817..ddbbfa3e5d42 100644
--- a/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml
+++ b/.github/workflows/beam_PostCommit_Java_SingleStoreIO_IT.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_SingleStoreIO_IT.json']
   workflow_dispatch:
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
 permissions:
diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml
index cc04e5e3b493..9704b7f03d61 100644
--- a/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Tpcds_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -57,7 +57,9 @@ env:
     --sourceType=PARQUET 
     --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned 
     --resultsDirectory=gs://beam-tpcds/results/dataflow/ 
-    --tpcParallel=1 
+    --tpcParallel=1
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
   tpcdsBigQueryArgs: |
     --bigQueryTable=tpcds 
     --bigQueryDataset=tpcds 
diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
index 55f7b65c9b5d..820a4c9792cc 100644
--- a/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Tpcds_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -55,6 +55,8 @@ env:
     --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned
     --resultsDirectory=gs://beam-tpcds/results/flink/
     --tpcParallel=1
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
   tpcdsBigQueryArgs: |
     --bigQueryTable=tpcds 
     --bigQueryDataset=tpcds 
diff --git a/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml b/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml
index 8bf21480f621..f39feaf64a1b 100644
--- a/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -48,12 +48,13 @@ env:
   GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
   GRADLE_COMMAND_ARGUMENTS: |
-    --runner=SparkRunner
     --dataSize=1GB
     --sourceType=PARQUET
     --dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned
     --resultsDirectory=gs://beam-tpcds/results/spark3-rdd/
     --tpcParallel=1
+  INFLUXDB_USER: ${{ secrets.INFLUXDB_USER }}
+  INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }}
   tpcdsBigQueryArgs: |
     --bigQueryTable=tpcds 
     --bigQueryDataset=tpcds 
@@ -78,11 +79,12 @@ jobs:
       github.event.comment.body == 'Run Spark Runner Tpcds Tests'
     runs-on: [self-hosted, ubuntu-20.04, main]
     timeout-minutes: 240
-    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) ${{ matrix.runner }}
     strategy:
       matrix:
         job_name: ["beam_PostCommit_Java_Tpcds_Spark"]
         job_phrase: ["Run Spark Runner Tpcds Tests"]
+        runner: [SparkRunner, SparkStructuredStreamingRunner]
     steps:
       - uses: actions/checkout@v4
       - name: Setup repository
@@ -90,13 +92,13 @@ jobs:
         with:
           comment_phrase: ${{ matrix.job_phrase }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) ${{ matrix.runner }}
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
-      - name: run PostCommit Java Tpcds Spark script
+      - name: run PostCommit Java Tpcds Spark (${{ matrix.runner }}) script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:java:testing:tpcds:run
           arguments: |
             -Ptpcds.runner=:runners:spark:3 \
-            "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --queries=${{env.tpcdsQueriesArg}}" \
+            "-Ptpcds.args=${{env.tpcdsBigQueryArgs}} ${{env.tpcdsInfluxDBArgs}} ${{ env.GRADLE_COMMAND_ARGUMENTS }} --runner=${{ matrix.runner }} --queries=${{env.tpcdsQueriesArg}}" \
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml
index 338b148ff3fb..d34847d5340f 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml
@@ -19,9 +19,9 @@ name: PostCommit Java ValidatesRunner Dataflow
 
 on:
   schedule:
-    - cron: '30 4/6 * * *'
+    - cron: '30 4/8 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -82,8 +82,8 @@ jobs:
           gradle-command: :runners:google-cloud-dataflow-java:validatesRunner
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml
index 86b6b32a4abb..91a55dc14007 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml
@@ -19,9 +19,9 @@ name: PostCommit Java ValidatesRunner Dataflow JavaVersions
 
 on:
   schedule:
-    - cron: '30 4/6 * * *'
+    - cron: '30 4/8 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -100,8 +100,8 @@ jobs:
             -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml
index abe21ac3f783..e10a4db9abf3 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml
@@ -19,9 +19,9 @@ name: PostCommit Java ValidatesRunner Dataflow Streaming
 
 on:
   schedule:
-    - cron: '30 4/6 * * *'
+    - cron: '30 4/8 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -82,8 +82,8 @@ jobs:
           gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerStreaming
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml
index 63625b48ea2f..adb5c9f9c777 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 6/8 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -82,8 +82,8 @@ jobs:
           gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerV2
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml
index b372c4c2acda..2024de57a1c6 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 6/8 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -82,8 +82,8 @@ jobs:
           gradle-command: :runners:google-cloud-dataflow-java:validatesRunnerV2Streaming
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml
index 16e21c9faa26..1d6eb82d9045 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Direct.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
       - name: run validatesRunner script
         run: ./gradlew :runners:direct-java:validatesRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml
index fafc821a2b20..da9ab6419ec6 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -95,8 +95,8 @@ jobs:
             -x compileTestJava \
             -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml
index 8171760f5528..22fd277470c8 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -80,8 +80,8 @@ jobs:
         with:
           gradle-command: :runners:flink:1.15:validatesRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml
index 6bc1b0ffa03c..764211e26484 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -98,8 +98,8 @@ jobs:
             -Dorg.gradle.java.home=$JAVA_HOME_11_X64 \
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml
index 8f8993f3108e..76136e8038d2 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Samza.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :runners:samza:validatesRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml
index ce67510b1e50..b4e4e25fa4bd 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :runners:spark:3:validatesRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml
index 7883218d262a..55e3c8a9ea7a 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :runners:spark:3:validatesStructuredStreamingRunnerBatch
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml
index 099f9ab1e724..510c797fef36 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -98,8 +98,8 @@ jobs:
             -Dorg.gradle.java.home=$JAVA_HOME_11_X64 \
           max-workers: 12
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml
index de3d4914052e..d3c80473dd2e 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :runners:twister2:validatesRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml
index eb2139c562e9..35d5ac738d64 100644
--- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml
+++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_ULR.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 4/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -78,8 +78,8 @@ jobs:
       - name: run ulrLoopbackValidatesRunner script
         run: ./gradlew :runners:portability:java:ulrLoopbackValidatesRunner
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Javadoc.yml b/.github/workflows/beam_PostCommit_Javadoc.yml
index 8e2a170dba63..61af8b12d2a4 100644
--- a/.github/workflows/beam_PostCommit_Javadoc.yml
+++ b/.github/workflows/beam_PostCommit_Javadoc.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Javadoc.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -84,7 +84,7 @@ jobs:
           path-to-root: sdks/java/javadoc/build/docs/javadoc
           base-url-path: https://beam.apache.org/releases/javadoc/current/
       - name: Upload Javadoc Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: Javadoc Results
           path: '**/sdks/java/javadoc/build/docs/javadoc/**'
\ No newline at end of file
diff --git a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml
index 2f066979681e..2c0a2f0c49e9 100644
--- a/.github/workflows/beam_PostCommit_PortableJar_Flink.yml
+++ b/.github/workflows/beam_PostCommit_PortableJar_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_PortableJar_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -83,7 +83,7 @@ jobs:
           arguments: |
             -PpythonVersion=3.8 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml
index 8b5bc031f7fa..d15c846cf501 100644
--- a/.github/workflows/beam_PostCommit_PortableJar_Spark.yml
+++ b/.github/workflows/beam_PostCommit_PortableJar_Spark.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_PortableJar_Spark.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -83,7 +83,7 @@ jobs:
           arguments: |
             -PpythonVersion=3.8 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python.yml b/.github/workflows/beam_PostCommit_Python.yml
index 0b22466f517b..80226779d4af 100644
--- a/.github/workflows/beam_PostCommit_Python.yml
+++ b/.github/workflows/beam_PostCommit_Python.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -98,10 +98,10 @@ jobs:
         env:
           CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PostCommit_Python_Arm.yml b/.github/workflows/beam_PostCommit_Python_Arm.yml
index 1a88c468a67c..7bd8d9cccc55 100644
--- a/.github/workflows/beam_PostCommit_Python_Arm.yml
+++ b/.github/workflows/beam_PostCommit_Python_Arm.yml
@@ -23,7 +23,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Arm.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -114,10 +114,10 @@ jobs:
           MULTIARCH_TAG: ${{ steps.set_tag.outputs.TAG }}
           USER: github-actions
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml
index ae07cb0273e0..fa106bbe8a8f 100644
--- a/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_Examples_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Examples_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -83,7 +83,7 @@ jobs:
             -PuseWheelDistribution \
             -PpythonVersion=3.11 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml
index b4b620e5dd1f..1d778c4cad47 100644
--- a/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Python_Examples_Direct.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Examples_Direct.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -90,7 +90,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml
index 1f334f6a9963..f7bc434a2055 100644
--- a/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Python_Examples_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Examples_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -90,7 +90,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml
index 6a33c63f24a0..0ea616635257 100644
--- a/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Python_Examples_Spark.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '0 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Examples_Spark.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -90,7 +90,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml
index 1c8daf4cffa3..49b337a8d6ae 100644
--- a/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml
+++ b/.github/workflows/beam_PostCommit_Python_MongoDBIO_IT.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_MongoDBIO_IT.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -82,7 +82,7 @@ jobs:
           arguments: |
             -PpythonVersion=3.11 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml
index f63461369be0..2ce1db22e102 100644
--- a/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Python_Nexmark_Direct.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Nexmark_Direct.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml
index 196dd1eaa84e..bb3bae69806c 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_ValidatesContainer_Dataflow.json']
   workflow_dispatch:
   issue_comment:
     types: [created]
@@ -96,7 +96,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml
index ac2179dd2521..c13df624e7af 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_ValidatesContainer_Dataflow_With_RC.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -95,7 +95,7 @@ jobs:
             -PtestRCDependencies=true \
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml
index 79bc303f1117..55e3a80e93ee 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -98,7 +98,7 @@ jobs:
             -PuseWheelDistribution \
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml
index b403f76b9f9e..adff5bd94535 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -92,7 +92,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
index 4229304278c0..286d294e3e5a 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Samza.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Samza.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -90,7 +90,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml
index 06db87f8fb76..1324325ab70f 100644
--- a/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml
+++ b/.github/workflows/beam_PostCommit_Python_ValidatesRunner_Spark.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_ValidatesRunner_Spark.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -90,7 +90,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
index dd899a538e9a..e5a2c1b39134 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '15 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -82,7 +82,7 @@ jobs:
           gradle-command: :sdks:python:test-suites:dataflow:gcpCrossLanguagePostCommit
           arguments: -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
index 33eb748a2f84..d4e57997da70 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Direct.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -81,7 +81,7 @@ jobs:
         with:
           gradle-command: :sdks:python:test-suites:direct:gcpCrossLanguagePostCommit
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
index 4d71e507fe32..66402fad0b8d 100644
--- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -84,7 +84,7 @@ jobs:
             -PuseWheelDistribution \
             -PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_SQL.yml b/.github/workflows/beam_PostCommit_SQL.yml
index eae7d4374d5b..4466633552fb 100644
--- a/.github/workflows/beam_PostCommit_SQL.yml
+++ b/.github/workflows/beam_PostCommit_SQL.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_SQL.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -79,8 +79,8 @@ jobs:
         with:
           gradle-command: :sqlPostCommit
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_Sickbay_Python.yml b/.github/workflows/beam_PostCommit_Sickbay_Python.yml
index b4d054f07a3d..7ebdbcb38416 100644
--- a/.github/workflows/beam_PostCommit_Sickbay_Python.yml
+++ b/.github/workflows/beam_PostCommit_Sickbay_Python.yml
@@ -21,7 +21,7 @@ on:
   schedule:
     - cron: '0 8 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Sickbay_Python.json']
   workflow_dispatch:
 
 # This allows a subsequently queued workflow run to interrupt previous runs
@@ -93,7 +93,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
index e40112f0c5b1..0cb4352c2fbb 100644
--- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml
+++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_TransformService_Direct.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -88,7 +88,7 @@ jobs:
             -PuseWheelDistribution \
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_Website_Test.yml b/.github/workflows/beam_PostCommit_Website_Test.yml
index ba372d223281..95e49faf13a5 100644
--- a/.github/workflows/beam_PostCommit_Website_Test.yml
+++ b/.github/workflows/beam_PostCommit_Website_Test.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Website_Test.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml
index ee90f9176f6d..de1b995bc649 100644
--- a/.github/workflows/beam_PostCommit_XVR_Direct.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_Direct.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -98,8 +98,8 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PskipNonPythonTask=false \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml
index 00c756a714f3..134efdf0dfc3 100644
--- a/.github/workflows/beam_PostCommit_XVR_Flink.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '30 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_Flink.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -99,8 +99,8 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PskipNonPythonTask=false \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml
index aab8a0e0a84f..82d284a829ad 100644
--- a/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_XVR_GoUsingJava_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_GoUsingJava_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -74,6 +74,11 @@ jobs:
         uses: ./.github/actions/setup-environment-action
         with:
           python-version: 3.8
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: GCloud Docker credential helper
+        run: |
+          gcloud auth configure-docker us.gcr.io
       - name: run XVR GoUsingJava Dataflow script
         env:
           USER: github-actions
@@ -82,8 +87,8 @@ jobs:
         with:
           gradle-command: :runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerGoUsingJava
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -93,4 +98,4 @@ jobs:
         with:
           commit: '${{ env.prsha || env.GITHUB_SHA }}'
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
-          files: '**/build/test-results/**/*.xml'
\ No newline at end of file
+          files: '**/build/test-results/**/*.xml'
diff --git a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml
index 113c51626078..02024805beb8 100644
--- a/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_XVR_JavaUsingPython_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_JavaUsingPython_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -86,8 +86,8 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml
index 6759930d5de8..4908b34da18e 100644
--- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -83,7 +83,7 @@ jobs:
           arguments: |
             -PpythonVersion=3.11 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml
index dda068049a17..a4dd9a9abe1e 100644
--- a/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml
+++ b/.github/workflows/beam_PostCommit_XVR_PythonUsingJava_Dataflow.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_PythonUsingJava_Dataflow.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -86,7 +86,7 @@ jobs:
           arguments: |
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PostCommit_XVR_Samza.yml b/.github/workflows/beam_PostCommit_XVR_Samza.yml
index b05b588e0cf8..b56872dd778b 100644
--- a/.github/workflows/beam_PostCommit_XVR_Samza.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Samza.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_Samza.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -98,8 +98,8 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PskipNonPythonTask=false \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PostCommit_XVR_Spark3.yml b/.github/workflows/beam_PostCommit_XVR_Spark3.yml
index 0742196a6960..68f9cb93b5f6 100644
--- a/.github/workflows/beam_PostCommit_XVR_Spark3.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Spark3.yml
@@ -19,7 +19,7 @@ on:
   schedule:
     - cron: '45 5/6 * * *'
   pull_request_target:
-    paths: ['release/trigger_all_tests.json']
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_XVR_Spark3.json']
   workflow_dispatch:
 
 #Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
@@ -98,8 +98,8 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PskipNonPythonTask=false \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PreCommit_CommunityMetrics.yml b/.github/workflows/beam_PreCommit_CommunityMetrics.yml
index 570dd5dc3354..778383a7f7db 100644
--- a/.github/workflows/beam_PreCommit_CommunityMetrics.yml
+++ b/.github/workflows/beam_PreCommit_CommunityMetrics.yml
@@ -22,7 +22,7 @@ on:
     paths: ['.test-infra/metrics/**', 'buildSrc/build.gradle.kts', '.github/workflows/beam_PreCommit_CommunityMetrics.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['.test-infra/metrics/**', 'buildSrc/build.gradle.kts', 'release/trigger_all_tests.json']
+    paths: ['.test-infra/metrics/**', 'buildSrc/build.gradle.kts', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_CommunityMetrics.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_GHA.yml b/.github/workflows/beam_PreCommit_GHA.yml
index 94811bc0f507..33edb12a14d5 100644
--- a/.github/workflows/beam_PreCommit_GHA.yml
+++ b/.github/workflows/beam_PreCommit_GHA.yml
@@ -22,7 +22,7 @@ on:
     paths: ['.github/**/*.yml']
   pull_request_target:
     branches: ['master', 'release-*' ]
-    paths: ['.github/**/*.yml', 'release/trigger_all_tests.json']
+    paths: ['.github/**/*.yml', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GHA.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_Go.yml b/.github/workflows/beam_PreCommit_Go.yml
index 7df1296ca124..2e268e21684e 100644
--- a/.github/workflows/beam_PreCommit_Go.yml
+++ b/.github/workflows/beam_PreCommit_Go.yml
@@ -22,7 +22,7 @@ on:
     paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_Go.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json']
+    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Go.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_GoPortable.yml b/.github/workflows/beam_PreCommit_GoPortable.yml
index 609340e8ded3..8d62e73d5b25 100644
--- a/.github/workflows/beam_PreCommit_GoPortable.yml
+++ b/.github/workflows/beam_PreCommit_GoPortable.yml
@@ -22,7 +22,7 @@ on:
     paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_GoPortable.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json']
+    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GoPortable.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_GoPrism.yml b/.github/workflows/beam_PreCommit_GoPrism.yml
index 52c5553cd623..4af064f45c42 100644
--- a/.github/workflows/beam_PreCommit_GoPrism.yml
+++ b/.github/workflows/beam_PreCommit_GoPrism.yml
@@ -22,7 +22,7 @@ on:
     paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', '.github/workflows/beam_PreCommit_GoPrism.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json']
+    paths: ['model/**', 'sdks/go.mod', 'sdks/go.sum', 'sdks/go/**', 'release/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_GoPrism.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_ItFramework.yml b/.github/workflows/beam_PreCommit_ItFramework.yml
index cf4cd2980176..a883330140ba 100644
--- a/.github/workflows/beam_PreCommit_ItFramework.yml
+++ b/.github/workflows/beam_PreCommit_ItFramework.yml
@@ -29,6 +29,7 @@ on:
     paths:
       - 'it/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_ItFramework.json'
   issue_comment:
     types: [created]
   schedule:
@@ -89,8 +90,8 @@ jobs:
       - name: run ItFrameworkPrecommit script
         run: ./gradlew -p it build
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PreCommit_Java.yml b/.github/workflows/beam_PreCommit_Java.yml
index fe694b4eab70..1614e9e64c05 100644
--- a/.github/workflows/beam_PreCommit_Java.yml
+++ b/.github/workflows/beam_PreCommit_Java.yml
@@ -26,7 +26,6 @@ on:
       - 'examples/kotlin/**'
       - 'release/**'
       - '.github/workflows/beam_PreCommit_Java.yml'
-      - '!sdks/java/extensions/avro/**'
       - '!sdks/java/extensions/sql/**'
       - '!sdks/java/io/amazon-web-services/**'
       - '!sdks/java/io/amazon-web-services2/**'
@@ -78,7 +77,7 @@ on:
       - 'examples/kotlin/**'
       - 'release/**'
       - 'release/trigger_all_tests.json'
-      - '!sdks/java/extensions/avro/**'
+      - '.github/trigger_files/beam_PreCommit_Java.json'
       - '!sdks/java/extensions/sql/**'
       - '!sdks/java/io/amazon-web-services/**'
       - '!sdks/java/io/amazon-web-services2/**'
@@ -154,13 +153,13 @@ env:
 jobs:
   beam_PreCommit_Java:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
-    timeout-minutes: 180
     runs-on: [self-hosted, ubuntu-20.04, main]
     strategy:
       fail-fast: false
       matrix:
         job_name: [beam_PreCommit_Java]
         job_phrase: [Run Java PreCommit]
+    timeout-minutes: 120
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
@@ -185,8 +184,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -198,7 +197,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml
index 16d6562f827f..f8af89a14ef2 100644
--- a/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.yml
@@ -38,6 +38,7 @@ on:
       - "sdks/java/io/common/**"
       - "sdks/java/core/src/main/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct.json'
       - "build.gradle"
       - "buildSrc/**"
       - "gradle/**"
@@ -117,8 +118,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -130,7 +131,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml
index f7c1d66e2f1e..a693902c8eb3 100644
--- a/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.yml
@@ -38,6 +38,7 @@ on:
       - "sdks/java/io/common/**"
       - "sdks/java/core/src/main/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Amazon-Web-Services_IO_Direct.json'
       - "build.gradle"
       - "buildSrc/**"
       - "gradle/**"
@@ -117,8 +118,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -130,7 +131,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml
index 8df097d8428f..4a56321bfe9d 100644
--- a/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Amqp_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/amqp/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Amqp_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml
index d32361708491..94b7c39b950b 100644
--- a/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Azure_IO_Direct.yml
@@ -38,6 +38,7 @@ on:
       - "sdks/java/io/common/**"
       - "sdks/java/core/src/main/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Azure_IO_Direct.json'
       - "build.gradle"
       - "buildSrc/**"
       - "gradle/**"
@@ -110,8 +111,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -123,7 +124,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml
index 04e1b10c23bd..daea023283a1 100644
--- a/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Cassandra_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/cassandra/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Cassandra_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml
index f23aad3ea455..daa275086330 100644
--- a/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml
@@ -31,6 +31,7 @@ on:
       - "sdks/java/io/hadoop-common/**"
       - "sdks/java/io/hadoop-format/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Cdap_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -96,8 +97,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -109,7 +110,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml
index f7c5ea1b8615..4ddac57a165b 100644
--- a/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/clickhouse/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Clickhouse_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml
index 496e0c15ea66..e1b381d5633e 100644
--- a/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/csv/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Csv_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml
index 97264ac146a5..5ca6b8afa68e 100644
--- a/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/debezium/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Debezium_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -101,8 +102,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -114,7 +115,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml
index f51e50aa8200..a6a17cebdef9 100644
--- a/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml
@@ -29,6 +29,7 @@ on:
       - "sdks/java/io/elasticsearch/**"
       - "sdks/java/io/elasticsearch-tests/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_ElasticSearch_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -105,8 +106,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -118,7 +119,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml
index b96e459470cf..b7f762ca808c 100644
--- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml
+++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml
@@ -39,6 +39,7 @@ on:
       - 'examples/kotlin/**'
       - 'release/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Examples_Dataflow.json'
   issue_comment:
     types: [created]
   schedule:
@@ -73,7 +74,7 @@ permissions:
 jobs:
   beam_PreCommit_Java_Examples_Dataflow:
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
-    timeout-minutes: 60
+    timeout-minutes: 75
     runs-on: [self-hosted, ubuntu-20.04, main]
     strategy:
       matrix:
@@ -112,8 +113,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml
index f482e22ec4a2..bd4f98cd8714 100644
--- a/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml
+++ b/.github/workflows/beam_PreCommit_Java_Examples_Dataflow_Java21.yml
@@ -37,6 +37,7 @@ on:
     - 'examples/kotlin/**'
     - 'release/**'
     - 'release/trigger_all_tests.json'
+    - '.github/trigger_files/beam_PreCommit_Java_Examples_Dataflow_Java21.json'
   issue_comment:
     types: [created]
   schedule:
@@ -76,7 +77,7 @@ jobs:
       matrix:
         job_name: ["beam_PreCommit_Java_Examples_Dataflow_Java21"]
         job_phrase: ["Run Java_Examples_Dataflow_Java21 PreCommit"]
-    timeout-minutes: 60
+    timeout-minutes: 75
     if: |
       github.event_name == 'push' ||
       github.event_name == 'pull_request_target' ||
@@ -120,8 +121,8 @@ jobs:
           -Pjava21Home=$JAVA_HOME_21_X64 \
         max-workers: 12
     - name: Archive JUnit Test Results
-      uses: actions/upload-artifact@v3
-      if: failure()
+      uses: actions/upload-artifact@v4
+      if: ${{ !success() }}
       with:
         name: JUnit Test Results
         path: "**/build/reports/tests/"
@@ -133,7 +134,7 @@ jobs:
         comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
         files: '**/build/test-results/**/*.xml'
     - name: Archive SpotBugs Results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: SpotBugs Results
         path: '**/build/reports/spotbugs/*.html'
\ No newline at end of file
diff --git a/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml
index 8ce77fef6416..7f467abc2749 100644
--- a/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_File-schema-transform_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/file-schema-transform/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_File-schema-transform_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -93,8 +94,8 @@ jobs:
             -PdisableCheckStyle=true \
             -Dfile.encoding=UTF-8 \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -106,7 +107,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml b/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml
index d5c244b2dbc9..b7ae39a1ee88 100644
--- a/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml
+++ b/.github/workflows/beam_PreCommit_Java_Flink_Versions.yml
@@ -31,6 +31,7 @@ on:
       - 'runners/flink/**'
       - 'release/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Flink_Versions.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
           arguments: |
             -PdisableSpotlessCheck=true -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
index 73dd9d63d872..eec0383c02a9 100644
--- a/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_GCP_IO_Direct.yml
@@ -37,6 +37,7 @@ on:
       - "runners/core-construction-java/**"
       - "runners/core-java/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_GCP_IO_Direct.json'
       - "sdks/java/core/src/main/**"
       - "sdks/java/extensions/arrow/**"
       - "sdks/java/extensions/google-cloud-platform-core/**"
@@ -114,8 +115,8 @@ jobs:
             -PdisableCheckStyle=true \
             -PenableJacocoReport \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -127,7 +128,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Google-ads_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Google-ads_IO_Direct.yml
index 7e0ca73e7efd..2ddce986fc8e 100644
--- a/.github/workflows/beam_PreCommit_Java_Google-ads_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Google-ads_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
     - "sdks/java/io/google-ads/**"
     - 'release/trigger_all_tests.json'
+    - '.github/trigger_files/beam_PreCommit_Java_Google-ads_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -90,8 +91,8 @@ jobs:
           -PdisableSpotlessCheck=true \
           -PdisableCheckStyle=true \
     - name: Archive JUnit Test Results
-      uses: actions/upload-artifact@v3
-      if: failure()
+      uses: actions/upload-artifact@v4
+      if: ${{ !success() }}
       with:
         name: JUnit Test Results
         path: "**/build/reports/tests/"
@@ -103,7 +104,7 @@ jobs:
         comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
         files: '**/build/test-results/**/*.xml'
     - name: Archive SpotBugs Results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: always()
       with:
         name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml
index 805561f3475d..52c91a26aa3e 100644
--- a/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_HBase_IO_Direct.yml
@@ -29,6 +29,7 @@ on:
       - "sdks/java/io/hbase/**"
       - "sdks/java/io/hadoop-common/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_HBase_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -94,8 +95,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -107,7 +108,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml
index bb9e5ee31f27..97096108ebd1 100644
--- a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml
@@ -29,6 +29,7 @@ on:
       - "sdks/java/io/hcatalog/**"
       - "sdks/java/io/hadoop-common/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_HCatalog_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -94,8 +95,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -107,7 +108,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml
index ea09a8df70d6..604c72471fa0 100644
--- a/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Hadoop_IO_Direct.yml
@@ -53,6 +53,7 @@ on:
       - "sdks/java/io/hadoop-common/**"
       - "sdks/java/io/hadoop-format/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Hadoop_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -132,8 +133,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -145,7 +146,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml
index 4a221a911009..831a1bec53fa 100644
--- a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml
@@ -29,6 +29,7 @@ on:
       - "sdks/java/io/common/**"
       - "sdks/java/core/src/main/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_IOs_Direct.json'
   issue_comment:
     types: [created]
   workflow_dispatch:
@@ -93,8 +94,8 @@ jobs:
             -PdisableCheckStyle=true \
             -Dfile.encoding=UTF-8 \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -106,7 +107,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml
index 52a03ceea204..80869d74b642 100644
--- a/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_InfluxDb_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/influxdb/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_InfluxDb_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml
index f328dd513c88..9a11d4e7ef35 100644
--- a/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_JDBC_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/jdbc/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_JDBC_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -99,8 +100,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -112,7 +113,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml
index 9abd63bb79c0..8f502223e85e 100644
--- a/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Jms_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/jms/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Jms_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -99,8 +100,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -112,7 +113,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml
index c73c384c7881..ac63a39380b2 100644
--- a/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Kafka_IO_Direct.yml
@@ -35,6 +35,7 @@ on:
       - "sdks/java/io/synthetic/**"
       - "sdks/java/io/expansion-service/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Kafka_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -101,8 +102,8 @@ jobs:
             -PdisableCheckStyle=true \
             --no-parallel \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -114,7 +115,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml
index 2aa4d4c10fd8..f76fec0bdd91 100644
--- a/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Kinesis_IO_Direct.yml
@@ -45,6 +45,7 @@ on:
       - "gradle.bat"
       - "settings.gradle.kts"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Kinesis_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -124,8 +125,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -137,7 +138,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml
index 25f90072db5c..8948bedf1384 100644
--- a/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Kudu_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/kudu/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Kudu_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml
index 34fa18a23c71..f0647f93c52b 100644
--- a/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_MongoDb_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/mongodb/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_MongoDb_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml
index 0b26f28e8f69..17c4127cb40d 100644
--- a/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Mqtt_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/mqtt/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Mqtt_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml
index ed6d6baf456b..a4e0ac744f41 100644
--- a/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Neo4j_IO_Direct.yml
@@ -29,6 +29,7 @@ on:
       - "sdks/java/io/neo4j/**"
       - "sdks/java/testing/test-utils/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Neo4j_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -101,8 +102,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -114,7 +115,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml
index 0ff7785650de..2cadc893fbc1 100644
--- a/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml
+++ b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Batch.yml
@@ -33,6 +33,7 @@ on:
       - 'runners/java-fn-execution/**'
       - 'sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_PVR_Flink_Batch.json'
   issue_comment:
     types: [created]
   schedule:
@@ -97,13 +98,13 @@ jobs:
         env:
           CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH }}
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
       - name: Upload test report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: java-code-coverage-report
           path: "**/build/test-results/**/*.xml"
diff --git a/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml
index 841c6dbdcbab..ae62fd93d977 100644
--- a/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml
+++ b/.github/workflows/beam_PreCommit_Java_PVR_Flink_Docker.yml
@@ -37,6 +37,7 @@ on:
       - 'runners/flink/**'
       - 'runners/java-fn-execution/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_PVR_Flink_Docker.json'
   issue_comment:
     types: [created]
   schedule:
@@ -102,8 +103,8 @@ jobs:
         env:
           CLOUDSDK_CONFIG: ${{ env.KUBELET_GCLOUD_CONFIG_PATH}}
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml
index 998a6bbbac04..e2430ae1bef9 100644
--- a/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Parquet_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/parquet/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Parquet_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml
index 00becbfd7ee6..dd3402c79773 100644
--- a/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Pulsar_IO_Direct.yml
@@ -38,6 +38,7 @@ on:
       - "sdks/java/io/common/**"
       - "sdks/java/core/src/main/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Pulsar_IO_Direct.json'
       - "build.gradle"
       - "buildSrc/**"
       - "gradle/**"
@@ -110,8 +111,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -123,7 +124,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml
index 7adbfe74bab5..f796f842311f 100644
--- a/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_RabbitMq_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/rabbitmq/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_RabbitMq_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml
index 255df740a57f..8e2483357cc6 100644
--- a/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Redis_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/redis/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Redis_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_RequestResponse_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_RequestResponse_IO_Direct.yml
index cfe721c1e948..a2fabd3debd0 100644
--- a/.github/workflows/beam_PreCommit_Java_RequestResponse_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_RequestResponse_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
     - "sdks/java/io/rrio/**"
     - 'release/trigger_all_tests.json'
+    - '.github/trigger_files/beam_PreCommit_Java_RequestResponse_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -90,8 +91,8 @@ jobs:
           -PdisableSpotlessCheck=true \
           -PdisableCheckStyle=true \
     - name: Archive JUnit Test Results
-      uses: actions/upload-artifact@v3
-      if: failure()
+      uses: actions/upload-artifact@v4
+      if: ${{ !success() }}
       with:
         name: JUnit Test Results
         path: "**/build/reports/tests/"
@@ -103,7 +104,7 @@ jobs:
         comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
         files: '**/build/test-results/**/*.xml'
     - name: Archive SpotBugs Results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: always()
       with:
         name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml
index 733921da7ad2..620ab5ccec17 100644
--- a/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_SingleStore_IO_Direct.yml
@@ -29,6 +29,7 @@ on:
       - "sdks/java/io/singlestore/**"
       - "sdks/java/testing/test-utils/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_SingleStore_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -94,8 +95,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -107,7 +108,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml
index c84f0026b726..8a3041bccbfb 100644
--- a/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Snowflake_IO_Direct.yml
@@ -31,6 +31,7 @@ on:
       - "sdks/java/extensions/google-cloud-platform-core/**"
       - "sdks/java/testing/test-utils/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Snowflake_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -103,8 +104,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -116,7 +117,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml
index f3142b41dd4d..2071f894de8c 100644
--- a/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Solr_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/solr/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Solr_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml b/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml
index 9665e9770bc0..d22cd8cad7e6 100644
--- a/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml
+++ b/.github/workflows/beam_PreCommit_Java_Spark3_Versions.yml
@@ -29,6 +29,7 @@ on:
     paths:
       - 'runners/spark/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Spark3_Versions.json'
   issue_comment:
     types: [created]
   schedule:
@@ -101,8 +102,8 @@ jobs:
           gradle-command: :runners:spark:3:sparkVersionsTest
           arguments: -PdisableSpotlessCheck=true
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
diff --git a/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml
index 7032cc62a62a..0106ae61ab8f 100644
--- a/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Splunk_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/splunk/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Splunk_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml
index da26e226e105..442eda5a8bc8 100644
--- a/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Thrift_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/thrift/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Thrift_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml
index 20eff37521e2..407ea3218288 100644
--- a/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml
+++ b/.github/workflows/beam_PreCommit_Java_Tika_IO_Direct.yml
@@ -27,6 +27,7 @@ on:
     paths:
       - "sdks/java/io/tika/**"
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Java_Tika_IO_Direct.json'
   issue_comment:
     types: [created]
   schedule:
@@ -92,8 +93,8 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -105,7 +106,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Kotlin_Examples.yml b/.github/workflows/beam_PreCommit_Kotlin_Examples.yml
index 13d533442575..2a804bb9de09 100644
--- a/.github/workflows/beam_PreCommit_Kotlin_Examples.yml
+++ b/.github/workflows/beam_PreCommit_Kotlin_Examples.yml
@@ -39,6 +39,7 @@ on:
       - 'examples/kotlin/**'
       - 'release/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Kotlin_Examples.json'
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_Portable_Python.yml b/.github/workflows/beam_PreCommit_Portable_Python.yml
index 169bdb74a649..2fa4905ca736 100644
--- a/.github/workflows/beam_PreCommit_Portable_Python.yml
+++ b/.github/workflows/beam_PreCommit_Portable_Python.yml
@@ -43,6 +43,7 @@ on:
       - 'sdks/python/**'
       - 'release/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Portable_Python.json'
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_Python.yml b/.github/workflows/beam_PreCommit_Python.yml
index 670f41875ffb..cb7660e04245 100644
--- a/.github/workflows/beam_PreCommit_Python.yml
+++ b/.github/workflows/beam_PreCommit_Python.yml
@@ -17,7 +17,7 @@ name: PreCommit Python
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python.json']
   issue_comment:
     types: [created]
   push:
@@ -99,10 +99,10 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PreCommit_PythonDocker.yml b/.github/workflows/beam_PreCommit_PythonDocker.yml
index 26ecf97c9b52..754fbac95d0d 100644
--- a/.github/workflows/beam_PreCommit_PythonDocker.yml
+++ b/.github/workflows/beam_PreCommit_PythonDocker.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Docker
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_PythonDocker.json']
   issue_comment:
     types: [created]
   push:
diff --git a/.github/workflows/beam_PreCommit_PythonDocs.yml b/.github/workflows/beam_PreCommit_PythonDocs.yml
index 2ada89147321..708aa669efda 100644
--- a/.github/workflows/beam_PreCommit_PythonDocs.yml
+++ b/.github/workflows/beam_PreCommit_PythonDocs.yml
@@ -18,7 +18,7 @@ name: PreCommit Python Docs
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: ["sdks/python/**", 'release/trigger_all_tests.json']
+    paths: ["sdks/python/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_PythonDocs.json']
   issue_comment:
     types: [created]
   push:
diff --git a/.github/workflows/beam_PreCommit_PythonFormatter.yml b/.github/workflows/beam_PreCommit_PythonFormatter.yml
index 17d9cd855138..8eaf99ba83a3 100644
--- a/.github/workflows/beam_PreCommit_PythonFormatter.yml
+++ b/.github/workflows/beam_PreCommit_PythonFormatter.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Formatter
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "sdks/python/apache_beam/**", 'release/trigger_all_tests.json']
+    paths: [ "sdks/python/apache_beam/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_PythonFormatter.json']
   issue_comment:
     types: [created]
   push:
diff --git a/.github/workflows/beam_PreCommit_PythonLint.yml b/.github/workflows/beam_PreCommit_PythonLint.yml
index 583c484e8e93..2cc469170d57 100644
--- a/.github/workflows/beam_PreCommit_PythonLint.yml
+++ b/.github/workflows/beam_PreCommit_PythonLint.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Lint
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: ["sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: ["sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_PythonLint.json']
   issue_comment:
     types: [created]
   push:
diff --git a/.github/workflows/beam_PreCommit_Python_Coverage.yml b/.github/workflows/beam_PreCommit_Python_Coverage.yml
index f0444013df59..244f37cc208c 100644
--- a/.github/workflows/beam_PreCommit_Python_Coverage.yml
+++ b/.github/workflows/beam_PreCommit_Python_Coverage.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Coverage
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_Coverage.json']
   issue_comment:
     types: [created]
   push:
@@ -89,7 +89,7 @@ jobs:
           arguments: |
             -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PreCommit_Python_Dataframes.yml b/.github/workflows/beam_PreCommit_Python_Dataframes.yml
index 822c10862ac7..2dd6f394c9ca 100644
--- a/.github/workflows/beam_PreCommit_Python_Dataframes.yml
+++ b/.github/workflows/beam_PreCommit_Python_Dataframes.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Dataframes
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_Dataframes.json']
   issue_comment:
     types: [created]
   push:
@@ -99,10 +99,10 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PreCommit_Python_Examples.yml b/.github/workflows/beam_PreCommit_Python_Examples.yml
index 36a0bde3ebdc..cb0479c218f7 100644
--- a/.github/workflows/beam_PreCommit_Python_Examples.yml
+++ b/.github/workflows/beam_PreCommit_Python_Examples.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Examples
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_Examples.json']
   issue_comment:
     types: [created]
   push:
@@ -99,10 +99,10 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PreCommit_Python_Integration.yml b/.github/workflows/beam_PreCommit_Python_Integration.yml
index d0af4932168e..461d7891a0a5 100644
--- a/.github/workflows/beam_PreCommit_Python_Integration.yml
+++ b/.github/workflows/beam_PreCommit_Python_Integration.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Integration
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: ["model/**", "sdks/python/**", "release/**", 'release/trigger_all_tests.json']
+    paths: ["model/**", "sdks/python/**", "release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_Integration.json']
   issue_comment:
     types: [created]
   push:
@@ -105,10 +105,10 @@ jobs:
             -PuseWheelDistribution \
             -PpythonVersion=${{ matrix.python_version }} \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
index 7e083bf84313..64609737cb25 100644
--- a/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
+++ b/.github/workflows/beam_PreCommit_Python_PVR_Flink.yml
@@ -30,6 +30,7 @@ on:
       - 'runners/java-fn-execution/**'
       - 'runners/reference/**'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Python_PVR_Flink.json'
   issue_comment:
     types: [created]
   push:
@@ -113,7 +114,7 @@ jobs:
           arguments: |
             -PpythonVersion=3.11 \
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: Python Test Results
diff --git a/.github/workflows/beam_PreCommit_Python_Runners.yml b/.github/workflows/beam_PreCommit_Python_Runners.yml
index f43c4eb47b6e..9d80f9cf5537 100644
--- a/.github/workflows/beam_PreCommit_Python_Runners.yml
+++ b/.github/workflows/beam_PreCommit_Python_Runners.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Runners
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_Runners.json']
   issue_comment:
     types: [created]
   push:
@@ -99,10 +99,10 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PreCommit_Python_Transforms.yml b/.github/workflows/beam_PreCommit_Python_Transforms.yml
index 6d06596ed4c8..f4895ec27c53 100644
--- a/.github/workflows/beam_PreCommit_Python_Transforms.yml
+++ b/.github/workflows/beam_PreCommit_Python_Transforms.yml
@@ -17,7 +17,7 @@ name: PreCommit Python Transforms
 on:
   pull_request_target:
     branches: [ "master", "release-*" ]
-    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json']
+    paths: [ "model/**","sdks/python/**","release/**", 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Python_Transforms.json']
   issue_comment:
     types: [created]
   push:
@@ -99,10 +99,10 @@ jobs:
             -PpythonVersion=${{ matrix.python_version }} \
             -PuseWheelDistribution
       - name: Archive Python Test Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: failure()
         with:
-          name: Python Test Results
+          name: Python ${{ matrix.python_version }} Test Results
           path: '**/pytest*.xml'
       - name: Publish Python Test Results
         uses: EnricoMi/publish-unit-test-result-action@v2
diff --git a/.github/workflows/beam_PreCommit_SQL.yml b/.github/workflows/beam_PreCommit_SQL.yml
index ac72f6b752da..b67addfee1c0 100644
--- a/.github/workflows/beam_PreCommit_SQL.yml
+++ b/.github/workflows/beam_PreCommit_SQL.yml
@@ -22,7 +22,7 @@ on:
     paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json']
+    paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_SQL.json']
   issue_comment:
     types: [created]
   schedule:
@@ -91,8 +91,8 @@ jobs:
             -PdisableCheckStyle=true \
             -PenableJacocoReport \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -104,7 +104,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_SQL_Java11.yml b/.github/workflows/beam_PreCommit_SQL_Java11.yml
index 9dadb4dcc2ff..da28e6201f1a 100644
--- a/.github/workflows/beam_PreCommit_SQL_Java11.yml
+++ b/.github/workflows/beam_PreCommit_SQL_Java11.yml
@@ -22,7 +22,7 @@ on:
     paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL_Java11.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json']
+    paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_SQL_Java11.json']
   issue_comment:
     types: [created]
   schedule:
@@ -92,7 +92,7 @@ jobs:
       - name: Build and Test
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sqlPreCommit
+          gradle-command: :sdks:java:extensions:sql:preCommit
           arguments: |
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
@@ -100,8 +100,8 @@ jobs:
             -PskipCheckerFramework \
             -Pjava11Home=$JAVA_HOME_11_X64 \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -113,7 +113,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_SQL_Java17.yml b/.github/workflows/beam_PreCommit_SQL_Java17.yml
index 8a1e88f7c654..ef0b758ddd5b 100644
--- a/.github/workflows/beam_PreCommit_SQL_Java17.yml
+++ b/.github/workflows/beam_PreCommit_SQL_Java17.yml
@@ -22,7 +22,7 @@ on:
     paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL_Java17.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json']
+    paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_SQL_Java17.json']
   issue_comment:
     types: [created]
   schedule:
@@ -90,7 +90,7 @@ jobs:
       - name: Build and Test
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sqlPreCommit
+          gradle-command: :sdks:java:extensions:sql:preCommit
           arguments: |
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
@@ -98,8 +98,8 @@ jobs:
             -PskipCheckerFramework \
             -Pjava17Home=$JAVA_HOME_17_X64 \
       - name: Archive JUnit Test Results
-        uses: actions/upload-artifact@v3
-        if: failure()
+        uses: actions/upload-artifact@v4
+        if: ${{ !success() }}
         with:
           name: JUnit Test Results
           path: "**/build/reports/tests/"
@@ -111,7 +111,7 @@ jobs:
           comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
           files: '**/build/test-results/**/*.xml'
       - name: Archive SpotBugs Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: SpotBugs Results
diff --git a/.github/workflows/beam_PreCommit_Spotless.yml b/.github/workflows/beam_PreCommit_Spotless.yml
index 6aa590e7556e..4ab1ade15847 100644
--- a/.github/workflows/beam_PreCommit_Spotless.yml
+++ b/.github/workflows/beam_PreCommit_Spotless.yml
@@ -36,6 +36,7 @@ on:
       - 'examples/kotlin/**'
       - '.test-infra/jenkins/'
       - 'release/trigger_all_tests.json'
+      - '.github/trigger_files/beam_PreCommit_Spotless.json'
   issue_comment:
     types: [created]
   schedule:
@@ -99,7 +100,7 @@ jobs:
         with:
           gradle-command: spotlessCheck checkStyleMain checkStyleTest :buildSrc:spotlessCheck
       - name: Upload test report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: java-code-coverage-report
           path: "**/build/reports/checkstyle/*.xml"
diff --git a/.github/workflows/beam_PreCommit_Typescript.yml b/.github/workflows/beam_PreCommit_Typescript.yml
index f3184dc0f90d..678b8d53efe1 100644
--- a/.github/workflows/beam_PreCommit_Typescript.yml
+++ b/.github/workflows/beam_PreCommit_Typescript.yml
@@ -24,7 +24,7 @@ on:
     paths: ['sdks/python/apache_beam/runners/interactive/extensions/**', '.github/workflows/beam_PreCommit_Typescript.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['sdks/python/apache_beam/runners/interactive/extensions/**', 'release/trigger_all_tests.json']
+    paths: ['sdks/python/apache_beam/runners/interactive/extensions/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Typescript.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_Website.yml b/.github/workflows/beam_PreCommit_Website.yml
index 848b4d44deec..0626596768f8 100644
--- a/.github/workflows/beam_PreCommit_Website.yml
+++ b/.github/workflows/beam_PreCommit_Website.yml
@@ -22,7 +22,7 @@ on:
     paths: ['website/**','.github/workflows/beam_PreCommit_Website.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['website/**', 'release/trigger_all_tests.json']
+    paths: ['website/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Website.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml b/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml
index c7b51f137290..4696d3c6b9c8 100644
--- a/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml
+++ b/.github/workflows/beam_PreCommit_Website_Stage_GCS.yml
@@ -22,7 +22,7 @@ on:
     paths: ['website/**','.github/workflows/beam_PreCommit_Website_Stage_GCS.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['website/**', 'release/trigger_all_tests.json']
+    paths: ['website/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Website_Stage_GCS.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PreCommit_Whitespace.yml b/.github/workflows/beam_PreCommit_Whitespace.yml
index da58d309f354..e5a13e3cb3cf 100644
--- a/.github/workflows/beam_PreCommit_Whitespace.yml
+++ b/.github/workflows/beam_PreCommit_Whitespace.yml
@@ -22,7 +22,7 @@ on:
     paths: ['**.md', '**.build.gradle', 'build.gradle.kts', '.github/workflows/beam_PreCommit_Whitespace.yml']
   pull_request_target:
     branches: ['master', 'release-*']
-    paths: ['**.md', '**.build.gradle', 'build.gradle.kts', 'release/trigger_all_tests.json']
+    paths: ['**.md', '**.build.gradle', 'build.gradle.kts', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_Whitespace.json']
   issue_comment:
     types: [created]
   schedule:
diff --git a/.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml b/.github/workflows/beam_Publish_BeamMetrics.yml
similarity index 97%
rename from .github/workflows/beam_PostCommit_BeamMetrics_Publish.yml
rename to .github/workflows/beam_Publish_BeamMetrics.yml
index b655eab8088c..53f62b3bf9d4 100644
--- a/.github/workflows/beam_PostCommit_BeamMetrics_Publish.yml
+++ b/.github/workflows/beam_Publish_BeamMetrics.yml
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: PostCommit BeamMetrics Publish
+name: Publish BeamMetrics
 
 on:
   schedule:
@@ -48,7 +48,7 @@ env:
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
-  beam_PostCommit_BeamMetrics_Publish:
+  beam_Publish_BeamMetrics:
     if: |
       github.event_name == 'push' ||
       github.event_name == 'workflow_dispatch' ||
diff --git a/.github/workflows/beam_PostCommit_Website_Publish.yml b/.github/workflows/beam_Publish_Website.yml
similarity index 95%
rename from .github/workflows/beam_PostCommit_Website_Publish.yml
rename to .github/workflows/beam_Publish_Website.yml
index ed1729021e98..726071641b7b 100644
--- a/.github/workflows/beam_PostCommit_Website_Publish.yml
+++ b/.github/workflows/beam_Publish_Website.yml
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: PostCommit Website Publish
+name: Publish Website
 
 on:
   schedule:
@@ -47,13 +47,13 @@ env:
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
-  beam_PostCommit_Website_Publish:
+  beam_Publish_Website:
     if: |
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam')
     runs-on: [self-hosted, ubuntu-20.04, main]
     timeout-minutes: 30
-    name: beam_PostCommit_Website_Publish
+    name: beam_Publish_Website
     steps:
       - uses: actions/checkout@v4
       - name: Setup environment
diff --git a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
index 6681669f5958..8871af45b50e 100644
--- a/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
+++ b/.github/workflows/beam_Python_ValidatesContainer_Dataflow_ARM.yml
@@ -109,9 +109,9 @@ jobs:
         env:
           MULTIARCH_TAG: ${{ steps.set_tag.outputs.TAG }}
           USER: github-actions
-      - name: Archive code coverage results
-        uses: actions/upload-artifact@v3
-        if: always()
+      - name: Archive Python Test Results
+        uses: actions/upload-artifact@v4
+        if: failure()
         with:
-          name: python-code-coverage-report
+          name: Python ${{ matrix.python_version }} Test Results
           path: "**/pytest*.xml"
diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml
index 6f1499a63769..8d27cea1d5d5 100644
--- a/.github/workflows/build_release_candidate.yml
+++ b/.github/workflows/build_release_candidate.yml
@@ -59,7 +59,7 @@ jobs:
           ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}"
           repository: apache/beam
       - name: Install Java 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -95,9 +95,13 @@ jobs:
     if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}}
     runs-on: ubuntu-latest
     steps:
-      - name: Validate and mask apache id/password
+      - name: Mask Apache Password
+        run: |
+          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
+          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
+          echo "::add-mask::$APACHE_PASS"
+      - name: Validate apache id/password
         run: |
-          echo "::add-mask::${{ github.event.inputs.APACHE_PASSWORD }}"
           if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
           then
             echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
@@ -107,7 +111,7 @@ jobs:
             echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
           fi
       - name: Install Java 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -149,7 +153,7 @@ jobs:
 
             svn add --force .
             svn status
-            svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username ${{ github.event.inputs.APACHE_ID }} --password ${{ github.event.inputs.APACHE_PASSWORD }} 
+            svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}" 
         
   stage_python_artifacts:
     if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}}
@@ -157,9 +161,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Validate and mask apache id/password
+      - name: Mask Apache Password
+        run: |
+          # Workaround for Actions bug - https://github.com/actions/runner/issues/643
+          APACHE_PASS=$(jq -r '.inputs.APACHE_PASSWORD' $GITHUB_EVENT_PATH)
+          echo "::add-mask::$APACHE_PASS"
+      - name: Mask apache id/password
         run: |
-          echo "::add-mask::${{ github.event.inputs.APACHE_PASSWORD }}"
           if [ "${{ github.event.inputs.APACHE_ID }}" == "" ]
           then
             echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/"
@@ -227,7 +235,7 @@ jobs:
           cd ..
           svn add --force python
           svn status
-          svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username ${{ github.event.inputs.APACHE_ID }} --password ${{ github.event.inputs.APACHE_PASSWORD }}
+          svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username "${{ github.event.inputs.APACHE_ID }}" --password "${{ github.event.inputs.APACHE_PASSWORD }}"
         
 
   stage_docker:
@@ -243,19 +251,19 @@ jobs:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@v1.3.0
       - name: Install Java 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '11'
       - name: Install Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - run: echo $JAVA_HOME
       - run: echo "JAVA11_HOME=${JAVA_HOME}" >> "$GITHUB_OUTPUT"
         id: export-java11
       - name: Install Java 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -298,7 +306,7 @@ jobs:
           token: ${{ github.event.inputs.BEAM_SITE_TOKEN }}
           ref: release-docs
       - name: Install Python 3.8
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - name: Install node
@@ -306,7 +314,7 @@ jobs:
         with:
           node-version: '16'
       - name: Install Java 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
@@ -338,7 +346,7 @@ jobs:
       - name: Add canonical link into javadocs
         uses: cicirello/javadoc-cleanup@v1
         with:
-          path-to-root: ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc
+          path-to-root: beam/sdks/java/javadoc/build/docs/javadoc
           base-url-path: https://beam.apache.org/releases/javadoc/current/
       - name: Consolidate Release Docs to beam-site branch with symlinks
         working-directory: beam-site
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
index 58d14c6fcd68..82924c3abcbd 100644
--- a/.github/workflows/build_wheels.yml
+++ b/.github/workflows/build_wheels.yml
@@ -89,7 +89,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Get tag
@@ -125,12 +125,12 @@ jobs:
         working-directory: ./sdks/python
         run: mv $(ls | grep apache-beam) apache-beam-source
       - name: Upload source as artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: source
           path: sdks/python/apache-beam-source
       - name: Upload compressed sources as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: source_zip
           path: sdks/python/dist
@@ -167,13 +167,13 @@ jobs:
         run: mv $(ls | grep apache-beam) apache-beam-source-rc
       - name: Upload RC source as artifact
         if: steps.is_rc.outputs.is_rc == 1
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: source_rc${{ steps.get_rc_version.outputs.RC_NUM }}
           path: sdks/python/apache-beam-source-rc
       - name: Upload compressed RC sources as artifacts
         if: steps.is_rc.outputs.is_rc == 1
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: source_zip_rc${{ steps.get_rc_version.outputs.RC_NUM }}
           path: sdks/python/dist
@@ -204,7 +204,7 @@ jobs:
     if: needs.check_env_variables.outputs.gcp-variables-set == 'true'
     steps:
       - name: Download compressed sources from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: source_zip
           path: source/
@@ -237,18 +237,18 @@ jobs:
             arch: aarch64
     steps:
     - name: Download python source distribution from artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: source
         path: apache-beam-source
     - name: Download Python SDK RC source distribution from artifacts
       if: ${{ needs.build_source.outputs.is_rc == 1 }}
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: source_rc${{ needs.build_source.outputs.rc_num }}
         path: apache-beam-source-rc
     - name: Install Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.8
     - uses: docker/setup-qemu-action@v1
@@ -278,9 +278,9 @@ jobs:
         done
       shell: bash
     - name: Upload wheels as artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: wheelhouse-${{ matrix.os_python.os }}
+        name: wheelhouse-${{ matrix.os_python.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }}
         path: apache-beam-source/wheelhouse/
     - name: Build RC wheels
       if: ${{ needs.build_source.outputs.is_rc == 1 }}
@@ -303,9 +303,9 @@ jobs:
       shell: bash
     - name: Upload RC wheels as artifacts
       if: ${{ needs.build_source.outputs.is_rc == 1 }}
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
-        name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.os_python.os }}
+        name: wheelhouse-rc${{ needs.build_source.outputs.rc_num }}-${{ matrix.os_python.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }}
         path: apache-beam-source-rc/wheelhouse/
 
   upload_wheels_to_gcs:
@@ -318,11 +318,15 @@ jobs:
     strategy:
       matrix:
         os : [ubuntu-latest, macos-latest, windows-latest]
+        arch: [auto]
+        include:
+          - os: "ubuntu-latest"
+            arch: aarch64
     steps:
     - name: Download wheels from artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
-        name: wheelhouse-${{ matrix.os }}
+        name: wheelhouse-${{ matrix.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }}
         path: wheelhouse/
     - name: Authenticate on GCP
       uses: google-github-actions/setup-gcloud@v0
diff --git a/.github/workflows/code_completion_plugin_tests.yml b/.github/workflows/code_completion_plugin_tests.yml
index 38ffd2fbd3f4..ca03932c9944 100644
--- a/.github/workflows/code_completion_plugin_tests.yml
+++ b/.github/workflows/code_completion_plugin_tests.yml
@@ -73,7 +73,7 @@ jobs:
 
       # Setup Java environment for the next steps
       - name: Setup Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '11'
@@ -109,7 +109,7 @@ jobs:
       # Collect Tests Result of failed tests
       - name: Collect Tests Result
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: tests-result
           path: ${{ github.workspace }}/build/reports/tests
diff --git a/.github/workflows/cut_release_branch.yml b/.github/workflows/cut_release_branch.yml
index 4201d6018c60..4c03e45913b4 100644
--- a/.github/workflows/cut_release_branch.yml
+++ b/.github/workflows/cut_release_branch.yml
@@ -21,28 +21,21 @@
 # To learn more about GitHub Actions in Apache Beam check the CI.md
 
 # Workflow used after https://github.com/apache/beam/commit/4183e747becebd18becee5fff547af365910fc9c
-# If help is needed debugging issues, you can view the cut_release_branch.sh and start_snapshot_build.sh scripts at that commit
+# If help is needed debugging issues, you can view the cut_release_branch.sh script at that commit
 # for guidance on how to do this manually.
-# (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/release/src/main/scripts/cut_release_branch.sh and 
-# https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/release/src/main/scripts/start_snapshot_build.sh).
+# (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/release/src/main/scripts/cut_release_branch.sh).
 name: Cut Release Branch
 on:
   workflow_dispatch:
     inputs:
       RELEASE_VERSION:
-        description: Beam version of current release
+        description: Beam version of current release (branch being cut)
         required: true
         default: '2.XX.0'
       NEXT_VERSION:
         description: Next release version
         required: true
         default: '2.XX.0'
-      JENKINS_USERNAME:
-        description: Username of the current Jenkins user. Used to update mass_comment.py with all Jenkins jobs.
-        required: true
-      JENKINS_TOKEN:
-        description: API Token for the current Jenkins user. Can be generated at https://ci-beam.apache.org/user/<user>/configure
-        required: true
       CREATE_RELEASE_BRANCH:
         description: Whether to cut the release branch. You shouldnt skip this unless it has already been completed successfully (yes/no)
         required: true
@@ -102,11 +95,6 @@ jobs:
       SCRIPT_DIR: ./release/src/main/scripts
       RELEASE: ${{ github.event.inputs.RELEASE_VERSION }}
     steps:
-      - name: Mask Jenkins token
-        run: |
-          echo "::add-mask::$JENKINS_TOKEN"
-        env:
-          JENKINS_TOKEN: ${{ github.event.inputs.JENKINS_TOKEN }}
       - name: Validate Next Version
         run: |
           if [[ $NEXT_RELEASE =~ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then
@@ -123,34 +111,6 @@ jobs:
           git config user.email actions@"$RUNNER_NAME".local
       - name: Install xmllint
         run: sudo apt-get install -y libxml2-utils
-      - name: Update mass_comment.py with new Jenkins jobs
-        run: |
-          rm release/src/main/scripts/jenkins_jobs.txt
-          for obj in $(curl https://ci-beam.apache.org/api/json | jq '.jobs[]' -c); do
-            echo $obj
-            url=$(echo $obj | jq '.url' -r || echo "")
-            name=$(echo $obj | jq '.name' -r || echo "")
-            if [ "$url" != "" ] && [ "$name" != "" ]; then
-            curl --user $JENKINS_USERNAME:$JENKINS_TOKEN $url/config.xml > /tmp/config.xml
-
-              if [ "$(xmllint --xpath 'string(//disabled)' /tmp/config.xml)" = "true" ]; then
-                echo "SKIPPING $url - disabled"
-              else
-                phrase=$(xmllint --xpath 'string(//triggers/org.jenkinsci.plugins.ghprb.GhprbTrigger/triggerPhrase)' /tmp/config.xml)
-                if [ "$phrase" != "" ]; then
-                  echo "Appending ${phrase},${name}"
-                  echo "${phrase},${name}" >> /tmp/result
-                fi
-              fi
-
-              rm /tmp/config.xml
-            fi
-          done
-
-          cat /tmp/result | sort | uniq | grep -i -E 'precommit|postcommit|validates|vr|example|test' | grep -v -i -E 'load|perf|website' >> release/src/main/scripts/jenkins_jobs.txt
-        env:
-          JENKINS_USERNAME: ${{ github.event.inputs.JENKINS_USERNAME }}
-          JENKINS_TOKEN: ${{ github.event.inputs.JENKINS_TOKEN }}
       - name: Update .asf.yaml to protect new release branch from force push
         run: |
           sed -i -e "s/master: {}/master: {}\n    release-${RELEASE}: {}/g" .asf.yaml
@@ -177,36 +137,6 @@ jobs:
       BRANCH_NAME: snapshot_build-${{ github.event.inputs.RELEASE_VERSION }}
       RELEASE_BRANCH: release-${{ github.event.inputs.RELEASE_VERSION }}
     steps:
-      - name: Install Hub
-        run: |
-          cd ~
-          wget https://github.com/github/hub/releases/download/v2.14.2/hub-linux-amd64-2.14.2.tgz
-          tar zvxvf hub-linux-amd64-2.14.2.tgz
-          sudo ./hub-linux-amd64-2.14.2/install
-          echo "eval "$(hub alias -s)"" >> ~/.bashrc
-      - uses: actions/checkout@v4
-      - name: Set git config
-        run: |
-          git config user.name $GITHUB_ACTOR
-          git config user.email actions@"$RUNNER_NAME".local
-      - name: Create Snapshot Branch
-        run: |
-          git remote add ${REMOTE_NAME} ${REMOTE_URL}
-          git checkout -b ${BRANCH_NAME}
-          touch empty_file.json
-          git add -A
-          git commit -m "Add empty file in order to create PR"
-          git push -f ${REMOTE_NAME}
-      - name: Create Pull Request
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Trigger snapshot build
         run: |
-          hub pull-request -b apache:${RELEASE_BRANCH} -F- <<<"[DO NOT MERGE]Start snapshot build for release process
-          
-          
-          Run Gradle Publish"
-          
-          echo "NOTE: If there is no jenkins job started, please comment generated PR with: Run Gradle Publish"
-          echo "Things remained you need to do manually after build successful:"
-          echo "1. Close this generated PR in github website."
-          echo "2. Delete your remote branch ${BRANCH_NAME} form your beam repo in github website."
+          echo "Working in progress. Please trigger 'Release Nightly Snapshot' GHA workflow manually."
diff --git a/.github/workflows/dask_runner_tests.yml b/.github/workflows/dask_runner_tests.yml
index 35c320086992..628cf2c34462 100644
--- a/.github/workflows/dask_runner_tests.yml
+++ b/.github/workflows/dask_runner_tests.yml
@@ -41,7 +41,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Build source
@@ -51,7 +51,7 @@ jobs:
         working-directory: ./sdks/python/dist
         run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz
       - name: Upload compressed sources as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: python_sdk_source
           path: sdks/python/dist/apache-beam-source.tar.gz
@@ -72,7 +72,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.params.py_ver }}
       - name: Install tox
@@ -89,7 +89,7 @@ jobs:
         working-directory: ./sdks/python
         run: tox -c tox.ini -e ${{ matrix.params.tox_env }}-win-dask
       - name: Upload test logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: pytest-${{matrix.os}}-${{matrix.params.py_ver}}
diff --git a/.github/workflows/deploy_release_candidate_pypi.yaml b/.github/workflows/deploy_release_candidate_pypi.yaml
index fd3994f658e8..8dafd520b1bf 100644
--- a/.github/workflows/deploy_release_candidate_pypi.yaml
+++ b/.github/workflows/deploy_release_candidate_pypi.yaml
@@ -31,8 +31,9 @@ jobs:
     steps:
     - name: Mask PyPi id/password
       run: |
-        echo "::add-mask::${{ github.event.inputs.PYPI_USER }}"
-        echo "::add-mask::${{ github.event.inputs.PYPI_PASSWORD }}"
+        # Workaround for Actions bug - https://github.com/actions/runner/issues/643
+        PYPI_PASSWORD=$(jq -r '.inputs.PYPI_PASSWORD' $GITHUB_EVENT_PATH)
+        echo "::add-mask::$PYPI_PASSWORD"
     - name: Checkout
       uses: actions/checkout@v4
     - name: Setup environment
@@ -86,4 +87,4 @@ jobs:
         echo "Will upload the following files to PyPI:"
         ls
 
-        twine upload * -u ${{ github.event.inputs.PYPI_USER }} -p ${{ github.event.inputs.PYPI_PASSWORD }}
+        twine upload * -u "${{ github.event.inputs.PYPI_USER }}" -p "${{ github.event.inputs.PYPI_PASSWORD }}"
diff --git a/.github/workflows/finalize_release.yml b/.github/workflows/finalize_release.yml
index 9a8b47ad3e42..f3eb544139c7 100644
--- a/.github/workflows/finalize_release.yml
+++ b/.github/workflows/finalize_release.yml
@@ -13,11 +13,8 @@ on:
       RC:
         description: Integer RC version for the release that we'd like to finalize (e.g. 3 for RC3)
         required: true
-      PYPI_USER:
-        description: PyPi username to perform the PyPi upload with
-        required: false
-      PYPI_PASSWORD:
-        description: PyPi password to perform the PyPi upload with
+      PYPI_API_TOKEN:
+        description: PyPi API token to perform the PyPi upload with
         required: false
       PUSH_DOCKER_ARTIFACTS:
         description: Whether to push SDK docker images to docker hub Apache organization. Should be yes unless you've already completed this step.
@@ -91,16 +88,15 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
-    - name: Mask and validate PyPi id/password
+    - name: Mask PyPi password
       run: |
-        echo "::add-mask::${{ github.event.inputs.PYPI_USER }}"
-        echo "::add-mask::${{ github.event.inputs.PYPI_PASSWORD }}"
-        if [ "${{ github.event.inputs.PYPI_USER }}" == "" ]
-        then
-          echo "Must provide a PyPi username to publish artifacts to PyPi"
-          exit 1
-        fi
-        if [ "${{ github.event.inputs.PYPI_PASSWORD }}" == "" ]
+        # Workaround for Actions bug - https://github.com/actions/runner/issues/643
+        PYPI_PASSWORD=$(jq -r '.inputs.PYPI_PASSWORD' $GITHUB_EVENT_PATH)
+        echo "::add-mask::$PYPI_PASSWORD"
+    - name: Validate PyPi id/password
+      run: |
+        echo "::add-mask::${{ github.event.inputs.PYPI_API_TOKEN }}"
+        if [ "${{ github.event.inputs.PYPI_API_TOKEN }}" == "" ]
         then
           echo "Must provide a PyPi password to publish artifacts to PyPi"
           exit 1
@@ -122,7 +118,7 @@ jobs:
         cd "dist.apache.org/repos/dist/dev/beam/${RELEASE}/python/"
         echo "Will upload the following files to PyPI:"
         ls
-        twine upload * -u ${{ github.event.inputs.PYPI_USER }} -p ${{ github.event.inputs.PYPI_PASSWORD }}
+        twine upload * -u __token__ -p "${{ github.event.inputs.PYPI_API_TOKEN }}"
 
   push_git_tags:
     if: ${{github.event.inputs.TAG_RELEASE == 'yes'}}
diff --git a/.github/workflows/go_tests.yml b/.github/workflows/go_tests.yml
index 02947eff5ca0..ae5960666363 100644
--- a/.github/workflows/go_tests.yml
+++ b/.github/workflows/go_tests.yml
@@ -36,14 +36,14 @@ concurrency:
   cancel-in-progress: true
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, ubuntu-20.04, main]
     name: Go Build
     steps:
       - name: Check out code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '1.21'
           cache-dependency-path: |
diff --git a/.github/workflows/java_tests.yml b/.github/workflows/java_tests.yml
index 1132ba1c196b..bcb073f5a977 100644
--- a/.github/workflows/java_tests.yml
+++ b/.github/workflows/java_tests.yml
@@ -47,7 +47,7 @@ jobs:
   check_gcp_variables:
     timeout-minutes: 5
     name: "Check GCP variables set"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, ubuntu-20.04, main]
     outputs:
       gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
     steps:
@@ -70,7 +70,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -82,42 +82,40 @@ jobs:
         with:
           java-version: 8
           go-version: 1.21
-      - name: Remove default github maven configuration
-        # This step is a workaround to avoid a decryption issue of Beam's
-        # net.linguica.gradle.maven.settings plugin and github's provided maven
-        # settings.xml file
-        run: rm ~/.m2/settings.xml
         # :sdks:java:core:test
       - name: Run :sdks:java:core:test
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p sdks/java/core/ test
+          gradle-command: test
+          arguments: -p sdks/java/core/
       - name: Upload test logs for :sdks:java:core:test
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: java_unit_tests-sdks-java-core-test-${{ matrix.os }}
           path: sdks/java/core/build/reports/tests/test
       # :sdks:java:harness:test
       - name: Run :sdks:java:harness:test
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p sdks/java/harness/ test
+          gradle-command: test
+          arguments: -p sdks/java/harness/
         if: always()
       - name: Upload test logs for :sdks:java:harness:test
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: java_unit_tests-sdks-java-harness-test-${{ matrix.os }}
           path: sdks/java/harness/build/reports/tests/test
       # :runners:core-java:test
       - name: Run :runners:core-java:test
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p runners/core-java/ test
+          gradle-command: test
+          arguments: -p runners/core-java/
         if: always()
       - name: Upload test logs for :runners:core-java:test
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: java_unit_tests-runners-core-java-test-${{ matrix.os }}
@@ -129,7 +127,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -141,19 +139,15 @@ jobs:
         with:
           java-version: 8
           go-version: 1.21
-      - name: Remove default github maven configuration
-        # This step is a workaround to avoid a decryption issue of Beam's
-        # net.linguica.gradle.maven.settings plugin and github's provided maven
-        # settings.xml file
-        run: rm ~/.m2/settings.xml
       - name: Run WordCount Unix
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p examples/ integrationTest --tests org.apache.beam.examples.WordCountIT
+          gradle-command: integrationTest
+          arguments: -p examples/ --tests org.apache.beam.examples.WordCountIT
             -DintegrationTestRunner=direct
-            -DintegrationTestPipelineOptions=["--runner=DirectRunner","--tempRoot=./tmp"]
+            -DintegrationTestPipelineOptions=[\"--runner=DirectRunner\",\"--tempRoot=./tmp\"]
       - name: Upload test logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: java_wordcount_direct_runner-${{matrix.os}}
@@ -167,7 +161,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest,windows-latest]
+        os: [[self-hosted, ubuntu-20.04, main], windows-latest]
     if: |
       needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && (
         (github.event_name == 'push' || github.event_name == 'schedule') ||
@@ -191,19 +185,15 @@ jobs:
           service_account_key: ${{ secrets.GCP_SA_KEY }}
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           export_default_credentials: true
-      - name: Remove default github maven configuration
-        # This step is a workaround to avoid a decryption issue of Beam's
-        # gradle-command-action plugin and github's provided maven
-        # settings.xml file
-        run: rm ~/.m2/settings.xml
       - name: Run WordCount
-        uses: ./.github/actions/gradle-command-action
+        uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          arguments: -p examples/ integrationTest --tests org.apache.beam.examples.WordCountIT
-            -DintegrationTestPipelineOptions=["--runner=DataflowRunner","--project=${{ secrets.GCP_PROJECT_ID }}","--tempRoot=gs://${{ secrets.GCP_TESTING_BUCKET }}/tmp/"]
+          gradle-command: integrationTest
+          arguments: -p examples/ --tests org.apache.beam.examples.WordCountIT
+            -DintegrationTestPipelineOptions=[\"--runner=DataflowRunner\",\"--project=${{ secrets.GCP_PROJECT_ID }}\",\"--tempRoot=gs://${{ secrets.GCP_TESTING_BUCKET }}/tmp/\"]
             -DintegrationTestRunner=dataflow
       - name: Upload test logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: java_wordcount_dataflow-${{matrix.os}}
diff --git a/.github/workflows/local_env_tests.yml b/.github/workflows/local_env_tests.yml
index 7dbff4feeb45..580ca7ed9f2b 100644
--- a/.github/workflows/local_env_tests.yml
+++ b/.github/workflows/local_env_tests.yml
@@ -46,10 +46,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '1.21'
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - name: "Installing local env dependencies"
@@ -64,10 +64,10 @@ jobs:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: '1.21'
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - name: "Installing local env dependencies"
diff --git a/.github/workflows/python_dependency_tests.yml b/.github/workflows/python_dependency_tests.yml
index 166899df90cb..1c4012029d56 100644
--- a/.github/workflows/python_dependency_tests.yml
+++ b/.github/workflows/python_dependency_tests.yml
@@ -35,7 +35,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.params.py_ver }}
       - name: Install base_image_requirements.txt
diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
index 0309329e84e1..a179939e4df2 100644
--- a/.github/workflows/python_tests.yml
+++ b/.github/workflows/python_tests.yml
@@ -45,7 +45,7 @@ jobs:
   check_gcp_variables:
     timeout-minutes: 5
     name: "Check GCP variables"
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, ubuntu-20.04, main]
     outputs:
       gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
     steps:
@@ -70,12 +70,12 @@ jobs:
       )
     needs:
       - check_gcp_variables
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, ubuntu-20.04, main]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Build source
@@ -85,7 +85,7 @@ jobs:
         working-directory: ./sdks/python/dist
         run: mv $(ls | grep "apache-beam.*tar\.gz") apache-beam-source.tar.gz
       - name: Upload compressed sources as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: python_sdk_source
           path: sdks/python/dist/apache-beam-source.tar.gz
@@ -107,7 +107,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.params.py_ver }}
       - name: Install tox
@@ -121,7 +121,7 @@ jobs:
         working-directory: ./sdks/python
         run: tox -c tox.ini run -e ${{ matrix.params.tox_env }}-win
       - name: Upload test logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: pytest-${{matrix.os}}-${{matrix.params.py_ver}}
@@ -133,13 +133,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest]
         python: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
       - name: Install requirements
@@ -158,21 +158,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [[self-hosted, ubuntu-20.04, main], macos-latest, windows-latest]
         python: ["3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
       - name: Install go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: '1.21'
       - name: Download source from artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: python_sdk_source
           path: apache-beam-source
diff --git a/.github/workflows/run_perf_alert_tool.yml b/.github/workflows/run_perf_alert_tool.yml
index bc59bd945fe2..1f623571acde 100644
--- a/.github/workflows/run_perf_alert_tool.yml
+++ b/.github/workflows/run_perf_alert_tool.yml
@@ -37,7 +37,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Authenticate on GCP
diff --git a/.github/workflows/run_rc_validation.yml b/.github/workflows/run_rc_validation.yml
index 35fb3ce05eae..15979a9e1acd 100644
--- a/.github/workflows/run_rc_validation.yml
+++ b/.github/workflows/run_rc_validation.yml
@@ -114,13 +114,13 @@ jobs:
        ref: ${{env.RC_TAG}}
 
     - name: Install Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{matrix.py_version}}
         
 
     - name: Setup Java JDK
-      uses: actions/setup-java@v3.8.0
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: 11
@@ -187,13 +187,13 @@ jobs:
       uses: azure/setup-kubectl@v3
 
     - name: Setup Java JDK
-      uses: actions/setup-java@v3.8.0
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: 11
 
     - name: Install Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{matrix.py_version}}
 
@@ -284,7 +284,7 @@ jobs:
       with:
         ref: ${{env.RC_TAG}}
     - name: Install Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.8'
 
@@ -360,7 +360,7 @@ jobs:
           ref: ${{env.RC_TAG}}
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_version}}
 
@@ -408,7 +408,7 @@ jobs:
           ref: ${{env.RC_TAG}}
 
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_version}}
 
@@ -460,7 +460,7 @@ jobs:
         with:
           ref: ${{env.RC_TAG}}
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_version}}
 
@@ -509,7 +509,7 @@ jobs:
         with:
           ref: ${{env.RC_TAG}}
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{matrix.py_version}}
 
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 959b2009aa4a..e16aa75b9730 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -28,7 +28,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: 'This pull request has been marked as stale due to 60 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the dev@beam.apache.org list. Thank you for your contributions.'
diff --git a/.github/workflows/tour_of_beam_backend.yml b/.github/workflows/tour_of_beam_backend.yml
index 3585bd0e670f..e3a016a4b5a7 100644
--- a/.github/workflows/tour_of_beam_backend.yml
+++ b/.github/workflows/tour_of_beam_backend.yml
@@ -42,7 +42,7 @@ jobs:
         working-directory: ./learning/tour-of-beam/backend
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
           # pin to the biggest Go version supported by Cloud Functions runtime
           go-version: '1.16'
diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml
index e71834f09ffb..0fdcfb070a22 100644
--- a/.github/workflows/typescript_tests.yml
+++ b/.github/workflows/typescript_tests.yml
@@ -83,7 +83,7 @@ jobs:
         with:
           node-version: '16'
       - name: Install Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Setup Beam Python
@@ -138,7 +138,7 @@ jobs:
         with:
           node-version: '16'
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.8
       - name: Setup Beam Python
diff --git a/.gitmodules b/.gitmodules
index fa6e30a8850a..3a6406c405f3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -7,6 +7,3 @@
 [submodule ".github/actions/github-push-action"]
 	path = .github/actions/github-push-action
 	url = https://github.com/ad-m/github-push-action
-[submodule ".github/actions/gradle-command-action"]
-	path = .github/actions/gradle-command-action
-	url = https://github.com/eskatos/gradle-command-action
diff --git a/.test-infra/jenkins/job_PostCommit_CommunityMetrics.groovy b/.test-infra/jenkins/job_PostCommit_CommunityMetrics.groovy
deleted file mode 100644
index b6b958a830a4..000000000000
--- a/.test-infra/jenkins/job_PostCommit_CommunityMetrics.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-
-job('beam_Prober_CommunityMetrics') {
-  description('Health check probes for the Community Metrics infrastructure')
-  commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-  commonJobProperties.enablePhraseTriggeringFromPullRequest(delegate,
-      'Community Metrics Prober',
-      'Run Community Metrics Prober')
-
-  commonJobProperties.setAutoJob(delegate)
-
-  commonJobProperties.enablePhraseTriggeringFromPullRequest(
-      delegate,
-      'Community Metrics Prober',
-      'Run Community Metrics Prober')
-
-  // Gradle goals for this job.
-  steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(':communityMetricsProber')
-      switches('--rerun-tasks')
-      commonJobProperties.setGradleSwitches(delegate)
-    }
-  }
-}
diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy
deleted file mode 100644
index 0d2a8612bdfe..000000000000
--- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Flink.groovy
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Flink',
-    'Run XVR_Flink PostCommit', 'Flink CrossLanguageValidatesRunner Tests', this) {
-      description('Runs the CrossLanguageValidatesRunner suite on the Flink runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion ->
-          shell("echo \"*** RUN CROSS-LANGUAGE FLINK USING PYTHON ${pythonVersion} ***\"")
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesCrossLanguageRunner")
-            commonJobProperties.setGradleSwitches(delegate)
-            switches("-PpythonVersion=${pythonVersion}")
-            // only run non-python task (e.g. GoUsingJava) once
-            switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
-          }
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy
deleted file mode 100644
index 764bf697cbee..000000000000
--- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_JavaUsingPython_Dataflow.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_JavaUsingPython_Dataflow',
-    'Run XVR_JavaUsingPython_Dataflow PostCommit', 'Dataflow CrossLanguageValidatesRunner JavaUsingPython Tests', this) {
-      description('Runs the CrossLanguageValidatesRunner JavaUsingPython suite on the Dataflow runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion ->
-          shell("echo \"*** RUN CROSS-LANGUAGE DATAFLOW JAVA WITH PYTHON EXTERNAL TRANSFORMS USING PYTHON ${pythonVersion} ***\"")
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(':runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerJavaUsingPython')
-            commonJobProperties.setGradleSwitches(delegate)
-            switches("-PpythonVersion=${pythonVersion}")
-          }
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJavaSQL_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJavaSQL_Dataflow.groovy
deleted file mode 100644
index 8c5d7f5be8fe..000000000000
--- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_PythonUsingJavaSQL_Dataflow.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow',
-    'Run XVR_PythonUsingJavaSQL_Dataflow PostCommit', 'Dataflow CrossLanguageValidatesRunner PythonUsingJavaSQL Tests', this) {
-      description('Runs the CrossLanguageValidatesRunner PythonUsingJavaSQL suite on the Dataflow runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        CROSS_LANGUAGE_VALIDATES_RUNNER_DATAFLOW_USING_SQL_PYTHON_VERSIONS.each { pythonVersion ->
-          shell("echo \"*** RUN CROSS-LANGUAGE DATAFLOW PYTHON WITH JAVA SQL TRANSFORMS USING PYTHON ${pythonVersion} ***\"")
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(':runners:google-cloud-dataflow-java:validatesCrossLanguageRunnerPythonUsingSql')
-            commonJobProperties.setGradleSwitches(delegate)
-            switches("-PpythonVersion=${pythonVersion}")
-          }
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Samza.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Samza.groovy
deleted file mode 100644
index 1319fdec1fcc..000000000000
--- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Samza.groovy
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-// This job runs the suite of ValidatesRunner tests against the Samza runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Samza',
-    'Run XVR_Samza PostCommit', 'Samza CrossLanguageValidatesRunner Tests', this) {
-      description('Runs the CrossLanguageValidatesRunner suite on the Samza runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion ->
-          shell("echo \"*** RUN CROSS-LANGUAGE SAMZA USING PYTHON ${pythonVersion} ***\"")
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(":runners:samza:job-server:validatesCrossLanguageRunner")
-            commonJobProperties.setGradleSwitches(delegate)
-            switches("-PpythonVersion=${pythonVersion}")
-            // only run non-python task (e.g. GoUsingJava) once
-            switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
-          }
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Spark3.groovy b/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Spark3.groovy
deleted file mode 100644
index eec4eac88278..000000000000
--- a/.test-infra/jenkins/job_PostCommit_CrossLanguageValidatesRunner_Spark3.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_XVR_Spark3',
-    'Run XVR_Spark3 PostCommit', 'Spark3 CrossLanguageValidatesRunner Tests', this) {
-      description('Runs the CrossLanguageValidatesRunner suite on the Spark3 runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion ->
-          shell("echo \"*** RUN CROSS-LANGUAGE SPARK3 USING PYTHON ${pythonVersion} ***\"")
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(':runners:spark:3:job-server:validatesCrossLanguageRunner')
-            commonJobProperties.setGradleSwitches(delegate)
-            switches("-PpythonVersion=${pythonVersion}")
-            // only run non-python task (e.g. GoUsingJava) once
-            switches("-PskipNonPythonTask=${pythonVersion != CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS[0]}")
-          }
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Go.groovy b/.test-infra/jenkins/job_PostCommit_Go.groovy
deleted file mode 100644
index 3d1c6a3b53ff..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Go.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This is the Go postcommit which runs a gradle build, and the current set
-// of postcommit tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go', 'Run Go PostCommit',
-    'Go PostCommit (\"Run Go PostCommit\")', this) {
-      description('Runs Go PostCommit tests against master.')
-      previousNames(/beam_PostCommit_Go_GradleBuild/)
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(
-          delegate,
-          'master',
-          300) // increased to 5 hours.
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':goPostCommit')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('--no-parallel')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Flink.groovy
deleted file mode 100644
index e0cd744dc434..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Flink.groovy
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Go integration tests against the Flink runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go_VR_Flink',
-    'Run Go Flink ValidatesRunner', 'Go Flink ValidatesRunner Tests', this) {
-      description('Runs Go integration tests on the Flink runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 100, true, 'beam', true, 40)
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:go:test:flinkValidatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java.groovy b/.test-infra/jenkins/job_PostCommit_Java.groovy
deleted file mode 100644
index cb3e0930dc89..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java', 'Run Java PostCommit',
-    'Java SDK Post Commit Tests', this) {
-
-      description('Runs PostCommit tests on the Java SDK.')
-      previousNames(/beam_PostCommit_Java_GradleBuild/)
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':javaPostCommit')
-          commonJobProperties.setGradleSwitches(delegate)
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Avro_Versions.groovy b/.test-infra/jenkins/job_PostCommit_Java_Avro_Versions.groovy
deleted file mode 100644
index ea07cb263928..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Avro_Versions.groovy
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java tests that depends on Avro against different Avro API versions
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Avro_Versions', 'Run PostCommit_Java_Avro_Versions',
-    'Java Avro Versions Post Commit Tests', this) {
-
-      description('Java Avro Versions Post Commit Tests')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":javaAvroVersionsTest")
-          commonJobProperties.setGradleSwitches(delegate)
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy
deleted file mode 100644
index 2ab27da0f1b0..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Dataflow.groovy
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_DataflowV1', 'Run PostCommit_Java_Dataflow',
-    'Dataflow Java Post Commit Tests', this) {
-
-      description('Dataflow Java Post Commit Tests')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":runners:google-cloud-dataflow-java:postCommit")
-          commonJobProperties.setGradleSwitches(delegate)
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Dataflow_Examples_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Dataflow_Examples_Java11.groovy
deleted file mode 100644
index 7e1f383ca3c5..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Dataflow_Examples_Java11.groovy
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import PostcommitJobBuilder
-import CommonJobProperties as commonJobProperties
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_Java11',
-    'Run Java examples on Dataflow Java 11', 'Google Cloud Dataflow Runner Examples Java 11', this) {
-
-      description('Runs the Java Examples suite on the Java 11 enabled Dataflow runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:examples:java11PostCommit')
-
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Dataflow_Examples_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Dataflow_Examples_Java17.groovy
deleted file mode 100644
index 14df1a6724e9..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Dataflow_Examples_Java17.groovy
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import PostcommitJobBuilder
-import CommonJobProperties as commonJobProperties
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_Java17',
-    'Run Java examples on Dataflow Java 17', 'Google Cloud Dataflow Runner Examples Java 17', this) {
-
-      description('Runs the Java Examples suite on the Java 17 enabled Dataflow runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:examples:java17PostCommit')
-
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy
deleted file mode 100644
index 5bfda20db3cf..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2.groovy
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import PostcommitJobBuilder
-import CommonJobProperties as commonJobProperties
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2',
-    'Run Java Examples on Dataflow Runner V2', 'Google Cloud Dataflow Runner V2 Examples', this) {
-
-      description('Runs the Java Examples suite on Dataflow runner V2.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest')
-
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy
deleted file mode 100644
index 6229f7c48a72..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import PostcommitJobBuilder
-import CommonJobProperties as commonJobProperties
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_java11',
-    'Run Java 11 Examples on Dataflow Runner V2', 'Google Cloud Dataflow Runner V2 Examples Java 11', this) {
-
-      description('Runs the Java Examples suite on Dataflow runner V2 using the Java 11 SDK harness.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest')
-
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-          switches '-PdisableSpotlessCheck=true'
-          switches '-PdisableCheckStyle=true'
-          switches '-PtestJavaVersion=11'
-          switches '-PskipCheckerFramework'
-          switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}"
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy
deleted file mode 100644
index 7e52a7e09789..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import PostcommitJobBuilder
-import CommonJobProperties as commonJobProperties
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_java17',
-    'Run Java 17 Examples on Dataflow Runner V2', 'Google Cloud Dataflow Runner V2 Examples Java 17', this) {
-
-      description('Runs the Java Examples suite on Dataflow runner V2 using the Java 17 SDK harness.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:examplesJavaRunnerV2IntegrationTest')
-
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-          switches '-PdisableSpotlessCheck=true'
-          switches '-PdisableCheckStyle=true'
-          switches '-PtestJavaVersion=17'
-          switches '-PskipCheckerFramework'
-          switches "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}"
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Direct.groovy
deleted file mode 100644
index dbea565102df..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Direct.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Java examples tests with DirectRunner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Direct',
-    'Run Java Examples_Direct', 'Java Direct Runner Examples', this) {
-
-      description('Run Java Examples on Direct Runner')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Execute shell command to run examples.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:direct:examplesIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Flink.groovy
deleted file mode 100644
index bf759bac04ec..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Flink.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Java examples tests with FlinkRunner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Flink',
-    'Run Java Examples_Flink', 'Java Flink Runner Examples', this) {
-
-      description('Run Java Examples on Flink Runner')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Execute shell command to run examples.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:examplesIntegrationTest")
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Spark.groovy
deleted file mode 100644
index f4af7dea6c0a..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Spark.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Java examples tests with SparkRunner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Spark',
-    'Run Java Examples_Spark', 'Java Spark Runner Examples', this) {
-
-      description('Run Java Examples on Spark Runner')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Execute shell command to run examples.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:examplesIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Hadoop_Versions.groovy b/.test-infra/jenkins/job_PostCommit_Java_Hadoop_Versions.groovy
deleted file mode 100644
index dfca212d64fb..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Hadoop_Versions.groovy
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Hadoop_Versions', 'Run PostCommit_Java_Hadoop_Versions',
-    'Java Hadoop Versions Post Commit Tests', this) {
-
-      description('Java Hadoop Versions Post Commit Tests')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":javaHadoopVersionsTest")
-          commonJobProperties.setGradleSwitches(delegate)
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java11.groovy
deleted file mode 100644
index e65bc273431a..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java11.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Dataflow_Java11', 'Run Jpms Dataflow Java 11 PostCommit',
-    'JPMS Java 11 Dataflow Post Commit Tests', this) {
-
-      description('Runs JPMS tests on Dataflow using the Java 11 SDK.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_11_HOME}")
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy
deleted file mode 100644
index f518985ca7a8..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Dataflow_Java17.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Dataflow_Java17', 'Run Jpms Dataflow Java 17 PostCommit',
-    'JPMS Java 17 Dataflow Post Commit Tests', this) {
-
-      description('Runs JPMS tests on Dataflow using the Java 17 SDK.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:jpms-tests:dataflowRunnerIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-PskipCheckerFramework")
-          switches("-PtestJavaVersion=17")
-          switches("-Pjava17Home=${commonJobProperties.JAVA_17_HOME}")
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java11.groovy
deleted file mode 100644
index 9faf78d5694a..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java11.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Direct_Java11', 'Run Jpms Direct Java 11 PostCommit',
-    'JPMS Java 11 direct runner  Post Commit Tests', this) {
-
-      description('Runs JPMS tests on the direct runner using the Java 11 SDK.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:jpms-tests:directRunnerIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_11_HOME}")
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy
deleted file mode 100644
index 04c31389ecbe..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Direct_Java17.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Direct_Java17', 'Run Jpms Direct Java 17 PostCommit',
-    'JPMS Java 17 direct runner  Post Commit Tests', this) {
-
-      description('Runs JPMS tests on the direct runner using the Java 17 SDK.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:jpms-tests:directRunnerIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-PskipCheckerFramework")
-          switches("-PtestJavaVersion=17")
-          switches("-Pjava17Home=${commonJobProperties.JAVA_17_HOME}")
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Flink_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Flink_Java11.groovy
deleted file mode 100644
index fcd9a0d4ea2f..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Flink_Java11.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Flink_Java11', 'Run Jpms Flink Java 11 PostCommit',
-    'JPMS Java 11 Flink Post Commit Tests', this) {
-
-      description('Runs JPMS tests on Flink using the Java 11 SDK.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:jpms-tests:flinkRunnerIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_11_HOME}")
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Spark_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Jpms_Spark_Java11.groovy
deleted file mode 100644
index 35a9ef961f95..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Jpms_Spark_Java11.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Jpms_Spark_Java11', 'Run Jpms Spark Java 11 PostCommit',
-    'JPMS Java 11 Spark Post Commit Tests', this) {
-
-      description('Runs JPMS tests on Spark using the Java 11 SDK.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:jpms-tests:sparkRunnerIntegrationTest')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_11_HOME}")
-          // Specify maven home on Jenkins, needed by Maven archetype integration tests.
-          switches('-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.4')
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy
deleted file mode 100644
index 868bc69a8468..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow.groovy
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow runner.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Dataflow',
-    'Dataflow Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Dataflow',
-    'Run Dataflow Runner Nexmark Tests', 'Dataflow Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'region' : 'us-central1',
-    'suite' : 'STRESS',
-    'numWorkers' : 4,
-    'maxNumWorkers' : 4,
-    'autoscalingAlgorithm' : 'NONE',
-    'nexmarkParallel' : 16,
-    'enforceEncodability' : true,
-    'enforceImmutability' : true
-  ]
-  Nexmark.standardJob(delegate, Runner.DATAFLOW, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2.groovy
deleted file mode 100644
index a8ea21876f91..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2.groovy
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-// This job runs the suite of Nexmark tests against the Dataflow runner V2.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Dataflow_V2',
-    'Dataflow Runner V2 Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner V2.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_DataflowV2',
-    'Run Dataflow Runner V2 Nexmark Tests', 'Dataflow Runner V2 Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner V2 against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'influxTags' : '{\\\"runnerVersion\\\":\\\"V2\\\",\\\"javaVersion\\\":\\\"8\\\"}',
-    'exportSummaryToBigQuery' : false,
-    'region' : 'us-central1',
-    'suite' : 'STRESS',
-    'numWorkers' : 4,
-    'maxNumWorkers' : 4,
-    'autoscalingAlgorithm' : 'NONE',
-    'nexmarkParallel' : 16,
-    'enforceEncodability' : true,
-    'enforceImmutability' : true
-  ]
-
-  def final JOB_SPECIFIC_SWITCHES = [
-    '-Pnexmark.runner.version="V2"'
-  ]
-
-  Nexmark.nonQueryLanguageJobs(delegate, Runner.DATAFLOW, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext, JOB_SPECIFIC_SWITCHES, Nexmark.DEFAULT_JAVA_RUNTIME_VERSION)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2_Java11.groovy
deleted file mode 100644
index 16866841cc72..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2_Java11.groovy
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-// This job runs the suite of Nexmark tests against the Dataflow runner V2.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Dataflow_V2_Java11',
-    'Dataflow Runner V2 Java 11 Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner V2 on Java 11.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_DataflowV2_Java11',
-    'Run Dataflow Runner V2 Java 11 Nexmark Tests', 'Dataflow Runner V2 Java 11 Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner V2 on Java 11 against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'influxTags' : '{\\\"runnerVersion\\\":\\\"V2\\\",\\\"javaVersion\\\":\\\"11\\\"}',
-    'exportSummaryToBigQuery' : false,
-    'region' : 'us-central1',
-    'suite' : 'STRESS',
-    'numWorkers' : 4,
-    'maxNumWorkers' : 4,
-    'autoscalingAlgorithm' : 'NONE',
-    'nexmarkParallel' : 16,
-    'enforceEncodability' : true,
-    'enforceImmutability' : true
-  ]
-
-  def final JOB_SPECIFIC_SWITCHES = [
-    '-Pnexmark.runner.version="V2"'
-  ]
-
-  Nexmark.nonQueryLanguageJobs(delegate, Runner.DATAFLOW, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext, JOB_SPECIFIC_SWITCHES, Nexmark.JAVA_11_RUNTIME_VERSION)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2_Java17.groovy
deleted file mode 100644
index 40bfb477b568..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Dataflow_V2_Java17.groovy
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-// This job runs the suite of Nexmark tests against the Dataflow runner V2.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Dataflow_V2_Java17',
-    'Dataflow Runner V2 Java 17 Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner V2 on Java 17.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_DataflowV2_Java17',
-    'Run Dataflow Runner V2 Java 17 Nexmark Tests', 'Dataflow Runner V2 Java 17 Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Dataflow runner V2 on Java 17 against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'influxTags' : '{\\\"runnerVersion\\\":\\\"V2\\\",\\\"javaVersion\\\":\\\"17\\\"}',
-    'exportSummaryToBigQuery' : false,
-    'region' : 'us-central1',
-    'suite' : 'STRESS',
-    'numWorkers' : 4,
-    'maxNumWorkers' : 4,
-    'autoscalingAlgorithm' : 'NONE',
-    'nexmarkParallel' : 16,
-    'enforceEncodability' : true,
-    'enforceImmutability' : true
-  ]
-
-  def final JOB_SPECIFIC_SWITCHES = [
-    '-Pnexmark.runner.version="V2"'
-  ]
-
-  Nexmark.nonQueryLanguageJobs(delegate, Runner.DATAFLOW, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext, JOB_SPECIFIC_SWITCHES, Nexmark.JAVA_17_RUNTIME_VERSION)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
deleted file mode 100644
index c441c1e73e39..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Direct.groovy
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Direct',
-    'Direct Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Direct runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Direct',
-    'Run Direct Runner Nexmark Tests', 'Direct Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Direct runner against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'suite' : 'SMOKE',
-    'enforceEncodability' : true,
-    'enforceImmutability' : true
-  ]
-  Nexmark.standardJob(delegate, Runner.DIRECT, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
deleted file mode 100644
index cdbb2ab8afec..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Flink.groovy
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Flink',
-    'Flink Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Flink runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf', true, 40)
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Flink',
-    'Run Flink Runner Nexmark Tests', 'Flink Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Flink runner against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam', true, 40)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'suite' : 'SMOKE',
-    'streamTimeout' : 60,
-  ]
-
-  Nexmark.standardJob(delegate, Runner.FLINK, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
deleted file mode 100644
index 7df873f63067..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Nexmark_Spark.groovy
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Spark runner.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Spark',
-    'Spark Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Spark runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Nexmark_Spark',
-    'Run Spark Runner Nexmark Tests', 'Spark Runner Nexmark Tests', this) {
-
-      description('Runs the Nexmark suite on the Spark runner against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'suite'        : 'SMOKE',
-    'streamTimeout': 60
-  ]
-  // Spark doesn't run streaming jobs, therefore run only batch variants.
-  Nexmark.batchOnlyJob(delegate, Runner.SPARK, SDK.JAVA, JOB_SPECIFIC_OPTIONS, triggeringContext)
-
-  def final SPARK_STRUCTURED_STREAMING_JOB_SPECIFIC_OPTIONS = [
-    'suite'        : 'SMOKE',
-    'streamTimeout': 60,
-    // Skip query 3 (SparkStructuredStreamingRunner does not support State/Timers yet)
-    'skipQueries'  : 3,
-  ]
-  Nexmark.batchOnlyJob(delegate, Runner.SPARK_STRUCTURED_STREAMING, SDK.JAVA, SPARK_STRUCTURED_STREAMING_JOB_SPECIFIC_OPTIONS, triggeringContext)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy
deleted file mode 100644
index fb3e24867f8c..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Flink_Streaming.groovy
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Flink_Streaming',
-    'Run Java Flink PortableValidatesRunner Streaming', 'Java Flink PortableValidatesRunner Streaming Tests', this) {
-      description('Runs the Java PortableValidatesRunner suite on the Flink runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesPortableRunnerStreaming")
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-        // TODO(https://github.com/apache/beam/issues/20668): Enable this test suite once we have support.
-        //gradle {
-        //  rootBuildScriptDir(commonJobProperties.checkoutDir)
-        //  tasks(":runners:flink:${CommonTestProperties.getFlinkVersion()}:job-server:validatesPortableRunnerStreamingCheckpoint")
-        //  commonJobProperties.setGradleSwitches(delegate)
-        //}
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Samza.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Samza.groovy
deleted file mode 100644
index 881d4e1ca85f..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Samza.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Java ValidatesRunner tests against the Samza runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Samza',
-    'Run Java Samza PortableValidatesRunner', 'Java Samza PortableValidatesRunner Tests', this) {
-      description('Runs the Java PortableValidatesRunner suite on the Samza runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:samza:job-server:validatesPortableRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark3_Streaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark3_Streaming.groovy
deleted file mode 100644
index 2164ab554b3b..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark3_Streaming.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Java ValidatesRunner tests against the Spark runner in streaming mode.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark3_Streaming',
-    'Run Java Spark v3 PortableValidatesRunner Streaming', 'Java Spark v3 PortableValidatesRunner Streaming Tests', this) {
-      description('Runs the Java PortableValidatesRunner suite on the Spark v3 runner in streaming mode.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:job-server:validatesPortableRunnerStreaming')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark_Batch.groovy b/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark_Batch.groovy
deleted file mode 100644
index ba08cc2e4404..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_PortableValidatesRunner_Spark_Batch.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Java ValidatesRunner tests against the Spark runner in batch mode.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_PVR_Spark_Batch',
-    'Run Java Spark PortableValidatesRunner Batch', 'Java Spark PortableValidatesRunner Batch Tests', this) {
-      description('Runs the Java PortableValidatesRunner suite on the Spark runner in batch mode.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:job-server:validatesPortableRunnerBatch')
-          tasks(':runners:spark:3:job-server:validatesPortableRunnerDocker')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Sickbay.groovy b/.test-infra/jenkins/job_PostCommit_Java_Sickbay.groovy
deleted file mode 100644
index 40e6ac61fa83..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Sickbay.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Java sickbay tests.
-this.buildSchedule = 'H H * * *'
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Sickbay',
-    'Run Java Sickbay', 'Java Sickbay Tests', this) {
-
-      description('Run Java Sickbay Tests')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Execute shell command to run sickbay tests.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':javaPostCommitSickbay')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy b/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy
deleted file mode 100644
index 901b76364333..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_SingleStoreIO_IT.groovy
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as common
-import PostcommitJobBuilder
-import Kubernetes
-
-String jobName = "beam_PostCommit_Java_SingleStoreIO_IT"
-
-void waitForPodWithLabel(job, Kubernetes k8s, String label) {
-  job.steps {
-    shell("${k8s.KUBERNETES_DIR}/singlestore/wait-for-pod-with-label.sh ${label} 600")
-  }
-}
-
-void waitFor(job, Kubernetes k8s, String resource) {
-  job.steps {
-    shell("${k8s.KUBERNETES_DIR}/singlestore/wait-for.sh ${resource} 600")
-  }
-}
-
-
-// This job runs the integration test of java SingleStoreIO class.
-PostcommitJobBuilder.postCommitJob(jobName,
-    'Run Java SingleStoreIO_IT', 'Java SingleStoreIO Integration Test',this) {
-      description('Runs the Java SingleStoreIO Integration Test.')
-
-      // Set common parameters.
-      common.setTopLevelMainJobProperties(delegate)
-
-      // Deploy SingleStoreDB cluster
-      String namespace = common.getKubernetesNamespace(jobName)
-      String kubeconfigPath = common.getKubeconfigLocationForNamespace(namespace)
-      Kubernetes k8s = Kubernetes.create(delegate, kubeconfigPath, namespace)
-
-      k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-rbac.yaml"))
-      k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-cluster-crd.yaml"))
-      k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-operator.yaml"))
-      waitForPodWithLabel(delegate, k8s, "sdb-operator")
-
-      k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-cluster.yaml"))
-      waitFor(delegate, k8s, "memsqlclusters.memsql.com")
-
-      String singlestoreHostName = "LOAD_BALANCER_IP"
-      k8s.loadBalancerIP("svc-sdb-cluster-ddl", singlestoreHostName)
-
-      // Define test options
-      Map pipelineOptions = [
-        tempRoot                  : 'gs://temp-storage-for-perf-tests',
-        project                   : 'apache-beam-testing',
-        runner                    : 'DataflowRunner',
-        singleStoreServerName     : "\$${singlestoreHostName}",
-        singleStoreUsername : "admin",
-        singleStorePassword : "secretpass",
-        singleStorePort: "3306",
-        numberOfRecords: "1000",
-      ]
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(common.checkoutDir)
-          common.setGradleSwitches(delegate)
-          switches("--info")
-          switches("-DintegrationTestPipelineOptions=\'${common.joinPipelineOptions(pipelineOptions)}\'")
-          switches("-DintegrationTestRunner=dataflow")
-          tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIODefaultMapperIT")
-          tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIOSchemaTransformIT")
-          tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIOConnectionAttributesIT")
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy
deleted file mode 100644
index f2152861be90..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Dataflow.groovy
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-import InfluxDBCredentialsHelper
-
-import static TpcdsDatabaseProperties.tpcdsBigQueryArgs
-import static TpcdsDatabaseProperties.tpcdsInfluxDBArgs
-import static TpcdsDatabaseProperties.tpcdsQueriesArg
-
-
-// This job runs the suite of Tpcds tests against the Dataflow runner.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Dataflow',
-    'Dataflow Runner Tpcds Tests', this) {
-      description('Runs the Tpcds suite on the Dataflow runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-      InfluxDBCredentialsHelper.useCredentials(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        shell('echo "*** RUN TPC-DS USING DATAFLOW RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:google-cloud-dataflow-java"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=DataflowRunner',
-                '--region=us-central1',
-                '--numWorkers=4',
-                '--maxNumWorkers=4',
-                '--autoscalingAlgorithm=NONE',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/dataflow/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-      }
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Dataflow',
-    'Run Dataflow Runner Tpcds Tests', 'Dataflow Runner Tpcds Tests', this) {
-
-      description('Runs the Tpcds suite on the Dataflow runner against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Gradle goals for this job.
-      steps {
-        shell('echo "*** RUN TPC-DS USING DATAFLOW RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:google-cloud-dataflow-java"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=DataflowRunner',
-                '--region=us-central1',
-                '--numWorkers=4',
-                '--maxNumWorkers=4',
-                '--autoscalingAlgorithm=NONE',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/dataflow/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Flink.groovy
deleted file mode 100644
index b6bff8cfffd3..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Flink.groovy
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-import InfluxDBCredentialsHelper
-
-import static TpcdsDatabaseProperties.tpcdsBigQueryArgs
-import static TpcdsDatabaseProperties.tpcdsInfluxDBArgs
-import static TpcdsDatabaseProperties.tpcdsQueriesArg
-
-// This job runs the Tpcds benchmark suite against the Flink runner.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Flink',
-    'Flink Runner Tpcds Tests', this) {
-      description('Runs the Tpcds suite on the Flink runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-      InfluxDBCredentialsHelper.useCredentials(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        shell('echo "*** RUN TPC-DS USING FLINK RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:flink:1.13"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=FlinkRunner',
-                '--parallelism=4',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/flink/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-      }
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Flink',
-    'Run Flink Runner Tpcds Tests', 'Flink Runner Tpcds Tests', this) {
-
-      description('Runs the Tpcds suite on the Flink runner against a Pull Request, on demand.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-      InfluxDBCredentialsHelper.useCredentials(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        shell('echo "*** RUN TPC-DS USING FLINK RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:flink:1.13"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=FlinkRunner',
-                '--parallelism=4',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/flink/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Spark.groovy
deleted file mode 100644
index 0bd28b232a9e..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_Tpcds_Spark.groovy
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-import InfluxDBCredentialsHelper
-
-import static TpcdsDatabaseProperties.tpcdsBigQueryArgs
-import static TpcdsDatabaseProperties.tpcdsInfluxDBArgs
-import static TpcdsDatabaseProperties.tpcdsQueriesArg
-
-// This job runs the Tpcds benchmark suite against the Spark runner.
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Spark',
-    'Spark Runner Tpcds Tests', this) {
-      description('Runs the Tpcds suite on the Spark runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-      InfluxDBCredentialsHelper.useCredentials(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        shell('echo "*** RUN TPC-DS USING SPARK 3 RDD RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:spark:3"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=SparkRunner',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/spark3-rdd/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-        shell('echo "*** RUN TPC-DS USING SPARK 3 DATASET RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:spark:3"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=SparkStructuredStreamingRunner',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/spark3-dataset/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-      }
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Java_Tpcds_Spark',
-    'Run Spark Runner Tpcds Tests', 'Spark Runner Tpcds Tests', this) {
-
-      description('Runs the Tpcds suite on the Spark runner against a Pull Request, on demand.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-      InfluxDBCredentialsHelper.useCredentials(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        shell('echo "*** RUN TPC-DS USING SPARK 3 RDD RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:spark:3"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=SparkRunner',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/spark3-rdd/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-        shell('echo "*** RUN TPC-DS USING SPARK 3 DATASET RUNNER ***"')
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:testing:tpcds:run')
-          commonJobProperties.setGradleSwitches(delegate)
-          switches('-Ptpcds.runner=":runners:spark:3"' +
-              ' -Ptpcds.args="' +
-              [
-                commonJobProperties.mapToArgString(tpcdsBigQueryArgs),
-                commonJobProperties.mapToArgString(tpcdsInfluxDBArgs),
-                '--runner=SparkStructuredStreamingRunner',
-                '--dataSize=1GB',
-                '--sourceType=PARQUET',
-                '--dataDirectory=gs://beam-tpcds/datasets/parquet/nonpartitioned',
-                '--resultsDirectory=gs://beam-tpcds/results/spark3-dataset/',
-                '--tpcParallel=1',
-                '--queries=' + tpcdsQueriesArg
-              ].join(' '))
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
deleted file mode 100644
index b0c0fa2c6e47..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow
-// runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflow',
-    'Run Dataflow ValidatesRunner', 'Google Cloud Dataflow Runner ValidatesRunner Tests (streaming/batch auto)', this) {
-
-      description('Runs the ValidatesRunner suite on the Dataflow runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 480)
-      previousNames(/beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle/)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:validatesRunner')
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy
deleted file mode 100644
index bb71f605cd4f..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11',
-    'Run Dataflow ValidatesRunner Java 11', 'Google Cloud Dataflow Runner ValidatesRunner Tests On Java 11', this) {
-
-      description('Runs the ValidatesRunner suite on the Dataflow runner with Java 11 worker harness.')
-
-      def JAVA_11_HOME = '/usr/lib/jvm/java-11-openjdk-amd64'
-      def JAVA_8_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 480)
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:testJar')
-          tasks(':runners:google-cloud-dataflow-java:worker:shadowJar')
-          switches("-Dorg.gradle.java.home=${JAVA_8_HOME}")
-        }
-
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:validatesRunner')
-          switches('-x shadowJar')
-          switches('-x shadowTestJar')
-          switches('-x compileJava')
-          switches('-x compileTestJava')
-          switches('-x jar')
-          switches('-x testJar')
-          switches('-x classes')
-          switches('-x testClasses')
-          switches("-Dorg.gradle.java.home=${JAVA_11_HOME}")
-
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java17.groovy
deleted file mode 100644
index 7bfcc256619f..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java17.groovy
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflow_Java17',
-    'Run Dataflow ValidatesRunner Java 17', 'Google Cloud Dataflow Runner ValidatesRunner Tests On Java 17', this) {
-
-      description('Runs the ValidatesRunner suite on the Dataflow runner with Java 17 worker harness.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 480)
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:testJar')
-          tasks(':runners:google-cloud-dataflow-java:worker:shadowJar')
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_8_HOME}")
-        }
-
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:validatesRunner')
-          switches('-x shadowJar')
-          switches('-x shadowTestJar')
-          switches('-x compileJava')
-          switches('-x compileTestJava')
-          switches('-x jar')
-          switches('-x testJar')
-          switches('-x classes')
-          switches('-x testClasses')
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_17_HOME}")
-
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
deleted file mode 100644
index 5d5c060175b9..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.groovy
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow
-// runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming',
-    'Run Dataflow Streaming ValidatesRunner', 'Google Cloud Dataflow Runner ValidatesRunner Tests (streaming)', this) {
-
-      description('Runs the ValidatesRunner suite on the Dataflow runner forcing streaming mode.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 720)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:validatesRunnerStreaming')
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2.groovy
deleted file mode 100644
index abb3c9f5b442..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow
-// runner V2.
-this.buildSchedule = 'H H/8 * * *'
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_VR_Dataflow_V2',
-    'Run Java Dataflow V2 ValidatesRunner', 'Google Cloud Dataflow Runner V2 Java ValidatesRunner Tests', this) {
-
-      description('Runs Java ValidatesRunner suite on the Dataflow runner V2.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 390)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:validatesRunnerV2')
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.groovy
deleted file mode 100644
index 3c4b5bf3a750..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.groovy
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow
-// runner V2.
-this.buildSchedule = 'H H/8 * * *'
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_VR_Dataflow_V2_Streaming',
-    'Run Java Dataflow V2 ValidatesRunner Streaming', 'Google Cloud Dataflow Runner V2 Java ValidatesRunner Tests (streaming)', this) {
-
-      description('Runs Java ValidatesRunner suite on the Dataflow runner V2 forcing streaming mode.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 510)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:google-cloud-dataflow-java:validatesRunnerV2Streaming')
-          // Increase parallel worker threads above processor limit since most time is
-          // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
-          // because each one launches a Dataflow job with about 3 mins of overhead.
-          // 3 x num_cores strikes a good balance between maxing out parallelism without
-          // overloading the machines.
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct.groovy
deleted file mode 100644
index 13173390be42..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests against the Direct
-// runner compiled with Java 8.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Direct',
-    'Run Direct ValidatesRunner', 'Direct Runner ValidatesRunner Tests for Java', this) {
-
-      description('Builds the Direct Runner and runs ValidatesRunner test suite in Java.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:direct-java:validatesRunner')
-        }
-      }
-
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy
deleted file mode 100644
index ee9a99d4d834..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java11.groovy
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests using Java 11 against the Direct
-// runner compiled with Java 8.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Direct_Java11',
-    'Run Direct ValidatesRunner Java 11', 'Direct Runner ValidatesRunner Tests for Java 11', this) {
-
-      description('Builds the Direct Runner with Java 8 and runs ValidatesRunner test suite in Java 11.')
-
-      def JAVA_11_HOME = '/usr/lib/jvm/java-11-openjdk-amd64'
-      def JAVA_8_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:direct-java:shadowJar')
-          tasks(':runners:direct-java:shadowTestJar')
-          switches("-Dorg.gradle.java.home=${JAVA_8_HOME}")
-        }
-
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:direct-java:validatesRunner')
-          switches("-Dorg.gradle.java.home=${JAVA_11_HOME}")
-          switches('-x shadowJar')
-          switches('-x shadowTestJar')
-          switches('-x compileJava')
-          switches('-x compileTestJava')
-        }
-      }
-
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java17.groovy
deleted file mode 100644
index 50681914cd5c..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Direct_Java17.groovy
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests using Java 17 against the Direct
-// runner compiled with Java 8.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Direct_Java17',
-    'Run Direct ValidatesRunner Java 17', 'Direct Runner ValidatesRunner Tests for Java 17', this) {
-
-      description('Builds the Direct Runner with Java 8 and runs ValidatesRunner test suite in Java 17.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:direct-java:shadowJar')
-          tasks(':runners:direct-java:shadowTestJar')
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_8_HOME}")
-        }
-
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:direct-java:validatesRunner')
-          switches("-Dorg.gradle.java.home=${commonJobProperties.JAVA_17_HOME}")
-          switches('-x shadowJar')
-          switches('-x shadowTestJar')
-          switches('-x compileJava')
-          switches('-x compileTestJava')
-        }
-      }
-
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy
deleted file mode 100644
index 44231c2b07ec..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Samza.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Samza runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Samza',
-    'Run Samza ValidatesRunner', 'Apache Samza Runner ValidatesRunner Tests', this) {
-      description('Runs the ValidatesRunner suite on the Samza runner.')
-      previousNames(/beam_PostCommit_Java_ValidatesRunner_Samza_Gradle/)
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:samza:validatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy
deleted file mode 100644
index 0cb6bd2d3710..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Spark runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark',
-    'Run Spark ValidatesRunner', 'Apache Spark Runner ValidatesRunner Tests', this) {
-      description('Runs the ValidatesRunner suite on the Spark runner.')
-      previousNames(/beam_PostCommit_Java_ValidatesRunner_Spark_Gradle/)
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:validatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.groovy
deleted file mode 100644
index 31d5d5cf07b4..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Spark runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming',
-    'Run Spark StructuredStreaming ValidatesRunner', 'Apache Spark SparkStructuredStreaming Runner ValidatesRunner Tests', this) {
-      description('Runs the ValidatesRunner suite on the Structured Streaming Spark runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:validatesStructuredStreamingRunnerBatch')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark_Java11.groovy
deleted file mode 100644
index d0da52927b8a..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Spark_Java11.groovy
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Spark_Java11',
-    'Run Spark ValidatesRunner Java 11', 'Apache Spark Runner ValidatesRunner Tests On Java 11', this) {
-
-      description('Runs the ValidatesRunner suite on the Spark runner with Java 11.')
-
-      def JAVA_11_HOME = '/usr/lib/jvm/java-11-openjdk-amd64'
-      def JAVA_8_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 270)
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:jar')
-          tasks(':runners:spark:3:testJar')
-          switches("-Dorg.gradle.java.home=${JAVA_8_HOME}")
-        }
-
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:spark:3:validatesRunner')
-          switches('-x shadowJar')
-          switches('-x shadowTestJar')
-          switches('-x compileJava')
-          switches('-x compileTestJava')
-          switches('-x jar')
-          switches('-x testJar')
-          switches('-x classes')
-          switches('-x testClasses')
-          switches("-Dorg.gradle.java.home=${JAVA_11_HOME}")
-
-          commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Twister2.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Twister2.groovy
deleted file mode 100644
index fa33ded11187..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Twister2.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of ValidatesRunner tests against the Twister2 runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_Twister2',
-    'Run Twister2 ValidatesRunner', 'Twister2 Runner ValidatesRunner Tests', this) {
-      description('Runs the ValidatesRunner suite on the Twister2 runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-      previousNames(/beam_PostCommit_Java_ValidatesRunner_Twister2_Gradle/)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:twister2:validatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_ULR_LOOPBACK.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_ULR_LOOPBACK.groovy
deleted file mode 100644
index 855e8ac5400e..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_ULR_LOOPBACK.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-// This job runs the suite of ValidatesRunner tests against the Direct
-// runner compiled with Java 8.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_ValidatesRunner_ULR',
-    'Run ULR Loopback ValidatesRunner', 'Universal Local Runner ValidatesRunner Tests for Java, LOOPBACK mode', this) {
-
-      description('Builds the Universal Local Runner and runs the Java ValidatesRunner test suite in LOOPBACK mode (no Docker).')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':runners:portability:java:ulrLoopbackValidatesRunner')
-        }
-      }
-
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Javadoc.groovy b/.test-infra/jenkins/job_PostCommit_Javadoc.groovy
deleted file mode 100644
index ae2ab53bd3fe..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Javadoc.groovy
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Javadoc', 'Run Javadoc PostCommit',
-    'Javadoc generation', this) {
-
-      description('PostCommit job that generates aggregated Javadoc')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      publishers {
-        archiveJavadoc {
-          javadocDir("src/sdks/java/javadoc/build/docs/javadoc")
-          keepAll(false)
-        }
-        recordIssues {
-          tools {
-            java()
-            javaDoc()
-          }
-          enabledForFailure(true)
-        }
-      }
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:java:javadoc:aggregateJavadoc')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
deleted file mode 100644
index 0c6f51f8be54..000000000000
--- a/.test-infra/jenkins/job_PostCommit_PortableJar_Flink.groovy
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// Tests creation and execution of portable pipeline Jars on the Flink runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Flink',
-    'Run PortableJar_Flink PostCommit', 'Flink Portable Jar Tests', this) {
-      description('Tests creation and execution of portable pipeline Jars on the Flink runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:portable:py38:testPipelineJarFlinkRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy b/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy
deleted file mode 100644
index 1f1963a9b2e4..000000000000
--- a/.test-infra/jenkins/job_PostCommit_PortableJar_Spark.groovy
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// Tests creation and execution of portable pipeline Jars on the Spark runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_PortableJar_Spark',
-    'Run PortableJar_Spark PostCommit', 'Spark Portable Jar Tests', this) {
-      description('Tests creation and execution of portable pipeline Jars on the Spark runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:portable:py38:testPipelineJarSparkRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Dataflow.groovy
deleted file mode 100644
index bd04d3e0c3a5..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Dataflow.groovy
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PhraseTriggeringPostCommitBuilder
-import CronJobBuilder
-import LoadTestsBuilder
-
-def chicagoTaxiJob = { scope ->
-  scope.description('Runs the Chicago Taxi Example on the Dataflow runner.')
-
-  // Set common parameters.
-  commonJobProperties.setTopLevelMainJobProperties(scope)
-
-  def pipelineOptions = [
-    num_workers          : 5,
-    autoscaling_algorithm: 'NONE',
-  ]
-
-  // Gradle goals for this job.
-  scope.steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      commonJobProperties.setGradleSwitches(delegate)
-      tasks(':sdks:python:test-suites:dataflow:chicagoTaxiExample')
-      switches('-PgcsRoot=gs://temp-storage-for-perf-tests/chicago-taxi')
-      switches("-PpipelineOptions=\"${LoadTestsBuilder.parseOptions(pipelineOptions)}\"")
-    }
-  }
-}
-
-PhraseTriggeringPostCommitBuilder.postCommitJob(
-    'beam_PostCommit_Python_Chicago_Taxi_Dataflow',
-    'Run Chicago Taxi on Dataflow',
-    'Chicago Taxi Example on Dataflow ("Run Chicago Taxi on Dataflow")',
-    this
-    ) {
-      chicagoTaxiJob(delegate)
-    }
-
-// TODO(https://github.com/apache/beam/issues/19973): Chicago Taxi Example doesn't work in Python 3.
-// Uncomment below once it is fixed.
-//
-// CronJobBuilder.cronJob(
-//     'beam_PostCommit_Python_Chicago_Taxi_Dataflow',
-//     'H H * * *',
-//     this
-//     ) {
-//       chicagoTaxiJob(delegate)
-//     }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy
deleted file mode 100644
index 6cf852a16893..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_Chicago_Taxi_Example_Flink.groovy
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CronJobBuilder
-import Flink
-import LoadTestsBuilder
-import PhraseTriggeringPostCommitBuilder
-
-import static LoadTestsBuilder.DOCKER_BEAM_JOBSERVER
-import static LoadTestsBuilder.DOCKER_CONTAINER_REGISTRY
-import static PythonTestProperties.CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION
-
-def chicagoTaxiJob = { scope ->
-  scope.description('Runs the Chicago Taxi Example on the Flink runner.')
-  commonJobProperties.setTopLevelMainJobProperties(scope, 'master', 120)
-
-  def numberOfWorkers = 5
-  def beamSdkDockerImage = "beam_python${CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION}_sdk:latest"
-
-  Flink flink = new Flink(scope, 'beam_PostCommit_Python_Chicago_Taxi_Flink')
-  flink.setUp(
-      [
-        "${DOCKER_CONTAINER_REGISTRY}/${beamSdkDockerImage}"
-      ],
-      numberOfWorkers,
-      "${DOCKER_BEAM_JOBSERVER}/beam_flink${CommonTestProperties.getFlinkVersion()}_job_server:latest")
-
-  def pipelineOptions = [
-    parallelism             : numberOfWorkers,
-    job_endpoint            : 'localhost:8099',
-    environment_options     : "docker_container_image=${DOCKER_CONTAINER_REGISTRY}/${beamSdkDockerImage}",
-    environment_type        : 'DOCKER',
-    execution_mode_for_batch: 'BATCH_FORCED',
-  ]
-
-  scope.steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(":sdks:python:test-suites:portable:py${CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION.replace('.', '')}:chicagoTaxiExample")
-      switches('-PgcsRoot=gs://temp-storage-for-perf-tests/chicago-taxi')
-      switches("-PpipelineOptions=\"${LoadTestsBuilder.parseOptions(pipelineOptions)}\"")
-    }
-  }
-}
-
-PhraseTriggeringPostCommitBuilder.postCommitJob(
-    'beam_PostCommit_Python_Chicago_Taxi_Flink',
-    'Run Chicago Taxi on Flink',
-    'Chicago Taxi Example on Flink ("Run Chicago Taxi on Flink")',
-    this
-    ) {
-      chicagoTaxiJob(delegate)
-    }
-
-// TODO(https://github.com/apache/beam/issues/19973): Chicago Taxi Example doesn't work in Python 3.
-// Uncomment below once it is fixed.
-//
-// CronJobBuilder.cronJob(
-//     'beam_PostCommit_Python_Chicago_Taxi_Flink',
-//     'H H * * *',
-//     this
-//     ) {
-//       chicagoTaxiJob(delegate)
-//     }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy
deleted file mode 100644
index 1280fcb4e233..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-
-// This job runs end-to-end cross language GCP IO tests with DataflowRunner.
-// Collects tests with the @pytest.mark.uses_gcp_java_expansion_service decorator
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Xlang_Gcp_Dataflow',
-    'Run Python_Xlang_Gcp_Dataflow PostCommit', 'Python_Xlang_Gcp_Dataflow (\"Run Python_Xlang_Gcp_Dataflow PostCommit\")', this) {
-      description('Runs end-to-end cross language GCP IO tests on the Dataflow runner. \"Run Python_Xlang_Gcp_Dataflow PostCommit\"')
-
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-
-      // Gradle goals for this job.
-
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":sdks:python:test-suites:dataflow:gcpCrossLanguagePostCommit")
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-PuseWheelDistribution")
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy
deleted file mode 100644
index e4bf771be1ae..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-
-// This job runs end-to-end cross language GCP IO tests with DirectRunner.
-// Collects tests with the @pytest.mark.uses_gcp_java_expansion_service decorator
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Xlang_Gcp_Direct',
-    'Run Python_Xlang_Gcp_Direct PostCommit', 'Python_Xlang_Gcp_Direct (\"Run Python_Xlang_Gcp_Direct PostCommit\")', this) {
-      description('Runs end-to-end cross language GCP IO tests on the Direct runner. \"Run Python_Xlang_Gcp_Direct PostCommit\"')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":sdks:python:test-suites:direct:gcpCrossLanguagePostCommit")
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy
deleted file mode 100644
index 8f91f8102edc..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Python examples tests with DataflowRunner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Dataflow',
-    'Run Python Examples_Dataflow', 'Python Dataflow Runner Examples', this) {
-
-      description('Runs the Python Examples with DataflowRunner')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Execute shell command to run examples.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":sdks:python:test-suites:dataflow:examplesPostCommit")
-          commonJobProperties.setGradleSwitches(delegate)
-          switches("-PuseWheelDistribution")
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy
deleted file mode 100644
index 6277106c8cbc..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Python examples tests with SparkRunner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Spark',
-    'Run Python Examples_Spark', 'Python Spark Runner Examples', this) {
-
-      description('Runs the Python Examples with Spark Runner')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Execute shell command to run examples.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(":sdks:python:test-suites:portable:sparkExamplesPostCommit")
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
-
-
diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy b/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy
deleted file mode 100644
index 9a66b51e5423..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import CommonJobProperties as common
-import Kubernetes
-
-String jobName = "beam_python_mongoio_load_test"
-
-job(jobName) {
-  common.setTopLevelMainJobProperties(delegate)
-  common.setAutoJob(delegate, 'H H/12 * * *')
-  common.enablePhraseTriggeringFromPullRequest(
-      delegate,
-      'Python MongoDBIO Load Test',
-      'Run Python MongoDBIO Load Test')
-
-  String namespace = common.getKubernetesNamespace(jobName)
-  String kubeconfigPath = common.getKubeconfigLocationForNamespace(namespace)
-  Kubernetes k8s = Kubernetes.create(delegate, kubeconfigPath, namespace)
-
-  k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/mongodb/load-balancer/mongo.yml"))
-  String mongoHostName = "LOAD_BALANCER_IP"
-  k8s.loadBalancerIP("mongo-load-balancer-service", mongoHostName)
-
-  Map pipelineOptions = [
-    temp_location: 'gs://temp-storage-for-perf-tests/loadtests',
-    project      : 'apache-beam-testing',
-    region       : 'us-central1',
-    mongo_uri    : "mongodb://root:uuinkkS@\$${mongoHostName}:27017",
-    num_documents: '1000000',
-    batch_size   : '10000',
-    runner       : 'DataflowRunner',
-    num_workers  : '5'
-  ]
-
-  steps {
-    gradle {
-      rootBuildScriptDir(common.checkoutDir)
-      common.setGradleSwitches(delegate)
-      switches("-Popts=\'${common.mapToArgString(pipelineOptions)}\'")
-      tasks(":sdks:python:test-suites:dataflow:mongodbioIT")
-    }
-  }
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Nexmark_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Python_Nexmark_Direct.groovy
deleted file mode 100644
index 6927dbe49cdb..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_Nexmark_Direct.groovy
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import CommonTestProperties.Runner
-import CommonTestProperties.SDK
-import CommonTestProperties.TriggeringContext
-import NexmarkBuilder as Nexmark
-import NoPhraseTriggeringPostCommitBuilder
-import PhraseTriggeringPostCommitBuilder
-
-NoPhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Python_Nexmark_Direct',
-    'Python Direct Runner Nexmark Tests', this) {
-
-      description('Runs the Python Nexmark suite on the Direct runner.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240, true, 'beam-perf')
-
-      commonJob(delegate, TriggeringContext.POST_COMMIT)
-    }
-
-PhraseTriggeringPostCommitBuilder.postCommitJob('beam_PostCommit_Python_Nexmark_Direct',
-    'Run Python Direct Runner Nexmark Tests', 'Python Direct Runner Nexmark Tests', this) {
-
-      description('Runs the Python Nexmark suite on the Direct runner against a Pull Request, on demand.')
-
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      commonJob(delegate, TriggeringContext.PR)
-    }
-
-private void commonJob(delegate, TriggeringContext triggeringContext) {
-  def final JOB_SPECIFIC_OPTIONS = [
-    'suite' : 'SMOKE',
-    'enforceEncodability' : true,
-    'enforceImmutability' : true
-  ]
-  Nexmark.standardPythonJob(delegate, Runner.DIRECT, SDK.PYTHON, JOB_SPECIFIC_OPTIONS, triggeringContext)
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy b/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy
deleted file mode 100644
index f9504ee811f3..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.ALL_SUPPORTED_VERSIONS
-
-// This job defines the Python postcommit sickbay tests.
-this.buildSchedule = 'H H * * *'
-ALL_SUPPORTED_VERSIONS.each { pythonVersion ->
-  def versionSuffix = pythonVersion.replace('.', '')
-  PostcommitJobBuilder.postCommitJob("beam_PostCommit_Sickbay_Python${versionSuffix}",
-      "Run Python ${pythonVersion} PostCommit Sickbay",
-      "Python${versionSuffix}_PC_Sickbay(\"Run Python ${pythonVersion} PostCommit Sickbay\")", this) {
-        description("Runs Python postcommit sickbay tests using Python ${pythonVersion}.")
-
-        // Set common parameters.
-        commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
-
-        publishers {
-          archiveJunit('**/pytest*.xml')
-        }
-
-        // Execute shell command to test Python SDK.
-        steps {
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(":sdks:python:test-suites:dataflow:py${versionSuffix}:postCommitSickbay")
-            commonJobProperties.setGradleSwitches(delegate)
-          }
-        }
-      }
-}
\ No newline at end of file
diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
deleted file mode 100644
index 254096448668..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS
-
-// This job runs the suite of Python ValidatesContainer tests against the
-// Dataflow runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
-    'Run Python Dataflow ValidatesContainer', 'Google Cloud Dataflow Runner Python ValidatesContainer Tests', this) {
-      description('Runs Python ValidatesContainer suite on the Dataflow runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Execute shell command to test Python SDK.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:dataflow:validatesContainerTests')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
-
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont_with_RC',
-    'Run Python RC Dataflow ValidatesContainer', 'Google Cloud Dataflow Runner Python ValidatesContainer Tests with RC Dependencies', this) {
-      description('Runs Python ValidatesContainer suite on the Dataflow runner by installing Release Candidates.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Execute shell command to test Python SDK.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:dataflow:validatesContainerTests')
-          switches('-PtestRCDependencies=true')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
deleted file mode 100644
index db052a0046ce..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Python ValidatesRunner tests against the
-// Dataflow runner V2.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python Dataflow ValidatesRunner',
-    'Google Cloud Dataflow Runner Python ValidatesRunner Tests', this) {
-      description('Runs Python ValidatesRunner suite on the Dataflow runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 200)
-
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Execute gradle task to test Python SDK.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:dataflow:validatesRunnerBatchTests')
-          tasks(':sdks:python:test-suites:dataflow:validatesRunnerStreamingTests')
-          switches('-PuseWheelDistribution')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy
deleted file mode 100644
index c49e3677ab8a..000000000000
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Samza.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Python ValidatesRunner tests against the Samza runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_VR_Samza',
-    'Run Python Samza ValidatesRunner', 'Python Samza ValidatesRunner Tests', this) {
-      description('Runs the Python ValidatesRunner suite on the Samza runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Publish all test results to Jenkins.
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:portable:samzaValidatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_SQL.groovy b/.test-infra/jenkins/job_PostCommit_SQL.groovy
deleted file mode 100644
index 9207cdbae17d..000000000000
--- a/.test-infra/jenkins/job_PostCommit_SQL.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_SQL', 'Run SQL PostCommit',
-    'SQL Post Commit Tests', this) {
-
-      description('Runs PostCommit tests for Beam SQL.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 240)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/build/test-results/**/*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sqlPostCommit')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
diff --git a/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy b/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy
deleted file mode 100644
index 03d29069a52c..000000000000
--- a/.test-infra/jenkins/job_PostCommit_TransformService_Direct.groovy
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-import static PythonTestProperties.CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS
-
-// This job runs multi-language pipelines using the Docker Compose based TransformService against the Direct runner.
-// Collects tests with the @pytest.mark.uses_transform_service decorator
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_TransformService_Direct',
-    'Run TransformService_Direct PostCommit', 'Direct TransformService Tests', this) {
-      description('Runs the TransformService suite on the Direct runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-      // Publish all test results to Jenkins
-      publishers {
-        archiveJunit('**/pytest*.xml')
-      }
-
-      // Gradle goals for this job.
-      steps {
-        CROSS_LANGUAGE_VALIDATES_RUNNER_PYTHON_VERSIONS.each { pythonVersion ->
-          shell("echo \"*** RUN TRANSFORM SERVICE Python SDK TESTS USING THE DIRECT RUNNER AND THE PYTHON VERSION ${pythonVersion} ***\"")
-          gradle {
-            rootBuildScriptDir(commonJobProperties.checkoutDir)
-            tasks(':sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava')
-            commonJobProperties.setGradleSwitches(delegate)
-            switches '-PtestJavaVersion=11'
-            switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}"
-            switches("-PuseWheelDistribution")
-            switches("-PpythonVersion=${pythonVersion}")
-          }
-        }
-      }
-    }
diff --git a/.test-infra/jupyter/README.md b/.test-infra/jupyter/README.md
index fc0a5c87f027..dfb1b95c619c 100644
--- a/.test-infra/jupyter/README.md
+++ b/.test-infra/jupyter/README.md
@@ -37,4 +37,4 @@ jupyter notebook  # Should open a browser window.
 # Pull Requests
 
 To minimize file size, diffs, and ease reviews, please clear all cell output
-(cell -> all output -> clear) before commiting.
+(cell -> all output -> clear) before committing.
diff --git a/.test-infra/mock-apis/go.mod b/.test-infra/mock-apis/go.mod
index cc65cfbaac76..306d9f53ee07 100644
--- a/.test-infra/mock-apis/go.mod
+++ b/.test-infra/mock-apis/go.mod
@@ -45,12 +45,12 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/api v0.128.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
diff --git a/.test-infra/mock-apis/go.sum b/.test-infra/mock-apis/go.sum
index a928e3dae2f1..13ca0c274e93 100644
--- a/.test-infra/mock-apis/go.sum
+++ b/.test-infra/mock-apis/go.sum
@@ -101,8 +101,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -144,8 +144,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -154,8 +154,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
diff --git a/CHANGES.md b/CHANGES.md
index 9318e85d477b..f59ee274343c 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -53,7 +53,7 @@
 * ([#X](https://github.com/apache/beam/issues/X)).
 -->
 
-# [2.53.0] - Unreleased
+# [2.54.0] - Unreleased
 
 ## Highlights
 
@@ -63,14 +63,12 @@
 ## I/Os
 
 * Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
-* TextIO now supports skipping multiple header lines (Java) ([#17990](https://github.com/apache/beam/issues/17990)).
-* Python GCSIO is now implemented with GCP GCS Client instead of apitools ([#25676](https://github.com/apache/beam/issues/25676))
-* Adding support for LowCardinality DataType in ClickHouse (Java) ([#29533](https://github.com/apache/beam/pull/29533)).
+* Adding support for Tuples DataType in ClickHouse (Java) ([#29715](https://github.com/apache/beam/pull/29715)).
+
 
 ## New Features / Improvements
 
 * X feature added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
-* The Python SDK now type checks `collections.abc.Collections` types properly. Some type hints that were erroneously allowed by the SDK may now fail. ([#29272](https://github.com/apache/beam/pull/29272))
 
 ## Breaking Changes
 
@@ -78,7 +76,7 @@
 
 ## Deprecations
 
-* Euphoria DSL is deprecated and will be removed in a future release (not before 2.56.0) ([#29451](https://github.com/apache/beam/issues/29451))
+* X behavior is deprecated and will be removed in X versions ([#X](https://github.com/apache/beam/issues/X)).
 
 ## Bugfixes
 
@@ -91,6 +89,43 @@
 
 * ([#X](https://github.com/apache/beam/issues/X)).
 
+# [2.53.0] - 2024-01-04
+
+## Highlights
+
+* Python streaming users that use 2.47.0 and newer versions of Beam should update to version 2.53.0, which fixes a known issue: ([#27330](https://github.com/apache/beam/issues/27330)).
+
+## I/Os
+
+* TextIO now supports skipping multiple header lines (Java) ([#17990](https://github.com/apache/beam/issues/17990)).
+* Python GCSIO is now implemented with GCP GCS Client instead of apitools ([#25676](https://github.com/apache/beam/issues/25676))
+* Added support for handling bad records to KafkaIO (Java) ([#29546](https://github.com/apache/beam/pull/29546))
+* Add support for generating text embeddings in MLTransform for Vertex AI and Hugging Face Hub models.([#29564](https://github.com/apache/beam/pull/29564))
+* NATS IO connector added (Go) ([#29000](https://github.com/apache/beam/issues/29000)).
+* Adding support for LowCardinality (Java) ([#29533](https://github.com/apache/beam/pull/29533)).
+
+## New Features / Improvements
+
+* The Python SDK now type checks `collections.abc.Collections` types properly. Some type hints that were erroneously allowed by the SDK may now fail. ([#29272](https://github.com/apache/beam/pull/29272))
+* Running multi-language pipelines locally no longer requires Docker.
+  Instead, the same (generally auto-started) subprocess used to perform the
+  expansion can also be used as the cross-language worker.
+* Framework for adding Error Handlers to composite transforms added in Java ([#29164](https://github.com/apache/beam/pull/29164)).
+* Python 3.11 images now include google-cloud-profiler ([#29561](https://github.com/apache/beam/pull/29651)).
+
+## Deprecations
+
+* Euphoria DSL is deprecated and will be removed in a future release (not before 2.56.0) ([#29451](https://github.com/apache/beam/issues/29451))
+
+## Bugfixes
+
+* (Python) Fixed sporadic crashes in streaming pipelines that affected some users of 2.47.0 and newer SDKs ([#27330](https://github.com/apache/beam/issues/27330)).
+* (Python) Fixed a bug that caused MLTransform to drop identical elements in the output PCollection ([#29600](https://github.com/apache/beam/issues/29600)).
+
+## Security Fixes
+
+* Upgraded to go 1.21.5 to build, fixing [CVE-2023-45285](https://security-tracker.debian.org/tracker/CVE-2023-45285) and [CVE-2023-39326](https://security-tracker.debian.org/tracker/CVE-2023-39326)
+
 # [2.52.0] - 2023-11-17
 
 ## Highlights
@@ -107,7 +142,7 @@ should handle this. ([#25252](https://github.com/apache/beam/issues/25252)).
 * Add `UseDataStreamForBatch` pipeline option to the Flink runner. When it is set to true, Flink runner will run batch
   jobs using the DataStream API. By default the option is set to false, so the batch jobs are still executed
   using the DataSet API.
-* `upload_graph` as one of the Experiments options for DataflowRunner is no longer required when the graph is larger than 10MB for Java SDK ([PR#28621](https://github.com/apache/beam/pull/28621).
+* `upload_graph` as one of the Experiments options for DataflowRunner is no longer required when the graph is larger than 10MB for Java SDK ([PR#28621](https://github.com/apache/beam/pull/28621)).
 * state amd side input cache has been enabled to a default of 100 MB. Use `--max_cache_memory_usage_mb=X` to provide cache size for the user state API and side inputs. (Python) ([#28770](https://github.com/apache/beam/issues/28770)).
 * Beam YAML stable release. Beam pipelines can now be written using YAML and leverage the Beam YAML framework which includes a preliminary set of IO's and turnkey transforms. More information can be found in the YAML root folder and in the [README](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/yaml/README.md).
 
@@ -131,6 +166,10 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 * Fixed [CVE-2023-39325](https://www.cve.org/CVERecord?id=CVE-2023-39325) (Java/Python/Go) ([#29118](https://github.com/apache/beam/issues/29118)).
 * Mitigated [CVE-2023-47248](https://nvd.nist.gov/vuln/detail/CVE-2023-47248)  (Python) [#29392](https://github.com/apache/beam/issues/29392).
 
+## Known issues
+
+* MLTransform drops the identical elements in the output PCollection. For any duplicate elements, a single element will be emitted downstream. ([#29600](https://github.com/apache/beam/issues/29600)).
+
 # [2.51.0] - 2023-10-03
 
 ## New Features / Improvements
@@ -165,6 +204,8 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 * Long-running Python pipelines might experience a memory leak: [#28246](https://github.com/apache/beam/issues/28246).
 * Python pipelines using BigQuery Storage Read API might need to pin `fastavro`
   dependency to 1.8.3 or earlier on some runners that don't use Beam Docker containers: [#28811](https://github.com/apache/beam/issues/28811)
+* MLTransform drops the identical elements in the output PCollection. For any duplicate elements, a single element will be emitted downstream. ([#29600](https://github.com/apache/beam/issues/29600)).
+
 
 # [2.50.0] - 2023-08-30
 
@@ -226,6 +267,7 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 * Beam Python containers rely on a version of Debian/aom that has several security vulnerabilities: [CVE-2021-30474](https://nvd.nist.gov/vuln/detail/CVE-2021-30474), [CVE-2021-30475](https://nvd.nist.gov/vuln/detail/CVE-2021-30475), [CVE-2021-30473](https://nvd.nist.gov/vuln/detail/CVE-2021-30473), [CVE-2020-36133](https://nvd.nist.gov/vuln/detail/CVE-2020-36133), [CVE-2020-36131](https://nvd.nist.gov/vuln/detail/CVE-2020-36131), [CVE-2020-36130](https://nvd.nist.gov/vuln/detail/CVE-2020-36130), and [CVE-2020-36135](https://nvd.nist.gov/vuln/detail/CVE-2020-36135)
 * Python SDK's cross-language Bigtable sink mishandles records that don't have an explicit timestamp set: [#28632](https://github.com/apache/beam/issues/28632). To avoid this issue, set explicit timestamps for all records before writing to Bigtable.
 * Python SDK worker start-up logs, particularly PIP dependency installations, that are not logged at warning or higher are suppressed. This suppression is reverted in 2.51.0.
+* MLTransform drops the identical elements in the output PCollection. For any duplicate elements, a single element will be emitted downstream. ([#29600](https://github.com/apache/beam/issues/29600)).
 
 # [2.49.0] - 2023-07-17
 
diff --git a/build.gradle.kts b/build.gradle.kts
index 59161809f37c..d4bd0fca4fc1 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -205,6 +205,9 @@ tasks.rat {
 
     // Ignore poetry autogenerated files.
     "**/poetry.lock",
+
+    // DuetAI training prompts
+    "learning/prompts/**/*.md",
   )
 
   // Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior
@@ -232,14 +235,14 @@ tasks.register("javaPreCommit") {
   dependsOn(":model:pipeline:build")
   dependsOn(":model:job-management:build")
   dependsOn(":model:fn-execution:build")
-  dependsOn(":runners:google-cloud-dataflow-java:worker:build")
   dependsOn(":sdks:java:core:buildNeeded")
 
   // Inline :sdks:java:core:buildDependents so we can carve out pieces at a time
   dependsOn(":beam-validate-runner:build")
   dependsOn(":examples:java:build")
+  dependsOn(":examples:java:preCommit")
   dependsOn(":examples:java:twitter:build")
-  dependsOn(":examples:kotlin:build")
+  dependsOn(":examples:java:twitter:preCommit")
   dependsOn(":examples:multi-language:build")
   dependsOn(":model:fn-execution:build")
   dependsOn(":model:job-management:build")
@@ -247,6 +250,7 @@ tasks.register("javaPreCommit") {
   dependsOn(":runners:core-construction-java:build")
   dependsOn(":runners:core-java:build")
   dependsOn(":runners:direct-java:build")
+  dependsOn(":runners:direct-java:needsRunnerTests")
   dependsOn(":runners:extensions-java:metrics:build")
   // lowest supported flink version
   var flinkVersions = project.ext.get("allFlinkVersions") as Array<*>
@@ -268,6 +272,7 @@ tasks.register("javaPreCommit") {
   dependsOn(":runners:spark:3:job-server:build")
   dependsOn(":runners:twister2:build")
   dependsOn(":sdks:java:build-tools:build")
+  dependsOn(":sdks:java:container:java8:docker")
   dependsOn(":sdks:java:core:build")
   dependsOn(":sdks:java:core:jmh:build")
   dependsOn(":sdks:java:expansion-service:build")
@@ -286,16 +291,6 @@ tasks.register("javaPreCommit") {
   dependsOn(":sdks:java:extensions:schemaio-expansion-service:build")
   dependsOn(":sdks:java:extensions:sketching:build")
   dependsOn(":sdks:java:extensions:sorter:build")
-  dependsOn(":sdks:java:extensions:sql:build")
-  dependsOn(":sdks:java:extensions:sql:datacatalog:build")
-  dependsOn(":sdks:java:extensions:sql:expansion-service:build")
-  dependsOn(":sdks:java:extensions:sql:hcatalog:build")
-  dependsOn(":sdks:java:extensions:sql:jdbc:build")
-  dependsOn(":sdks:java:extensions:sql:perf-tests:build")
-  dependsOn(":sdks:java:extensions:sql:shell:build")
-  dependsOn(":sdks:java:extensions:sql:udf-test-provider:build")
-  dependsOn(":sdks:java:extensions:sql:udf:build")
-  dependsOn(":sdks:java:extensions:sql:zetasql:build")
   dependsOn(":sdks:java:extensions:timeseries:build")
   dependsOn(":sdks:java:extensions:zetasketch:build")
   dependsOn(":sdks:java:fn-execution:build")
@@ -309,6 +304,7 @@ tasks.register("javaPreCommit") {
   dependsOn(":sdks:java:io:sparkreceiver:2:build")
   dependsOn(":sdks:java:io:synthetic:build")
   dependsOn(":sdks:java:io:xml:build")
+  dependsOn(":sdks:java:javadoc:allJavadoc")
   dependsOn(":sdks:java:testing:expansion-service:build")
   dependsOn(":sdks:java:testing:jpms-tests:build")
   dependsOn(":sdks:java:testing:load-tests:build")
@@ -318,13 +314,6 @@ tasks.register("javaPreCommit") {
   dependsOn(":sdks:java:testing:watermarks:build")
   dependsOn(":sdks:java:transform-service:build")
   dependsOn(":sdks:java:transform-service:launcher:build")
-
-  dependsOn(":examples:java:preCommit")
-  dependsOn(":examples:java:twitter:preCommit")
-  dependsOn(":sdks:java:extensions:sql:jdbc:preCommit")
-  dependsOn(":sdks:java:javadoc:allJavadoc")
-  dependsOn(":runners:direct-java:needsRunnerTests")
-  dependsOn(":sdks:java:container:java8:docker")
 }
 
 // a precommit task build multiple IOs (except those splitting into single jobs)
@@ -350,6 +339,7 @@ tasks.register("javaioPreCommit") {
   dependsOn(":sdks:java:io:jdbc:build")
   dependsOn(":sdks:java:io:jms:build")
   dependsOn(":sdks:java:io:kafka:build")
+  dependsOn(":sdks:java:io:kafka:upgrade:build")
   dependsOn(":sdks:java:io:kudu:build")
   dependsOn(":sdks:java:io:mongodb:build")
   dependsOn(":sdks:java:io:mqtt:build")
@@ -376,10 +366,18 @@ tasks.register("flinkPreCommit") {
 }
 
 tasks.register("sqlPreCommit") {
-  dependsOn(":sdks:java:extensions:sql:runBasicExample")
-  dependsOn(":sdks:java:extensions:sql:runPojoExample")
-  dependsOn(":sdks:java:extensions:sql:build")
+  dependsOn(":sdks:java:extensions:sql:preCommit")
   dependsOn(":sdks:java:extensions:sql:buildDependents")
+  dependsOn(":sdks:java:extensions:sql:datacatalog:build")
+  dependsOn(":sdks:java:extensions:sql:expansion-service:build")
+  dependsOn(":sdks:java:extensions:sql:hcatalog:build")
+  dependsOn(":sdks:java:extensions:sql:jdbc:build")
+  dependsOn(":sdks:java:extensions:sql:jdbc:preCommit")
+  dependsOn(":sdks:java:extensions:sql:perf-tests:build")
+  dependsOn(":sdks:java:extensions:sql:shell:build")
+  dependsOn(":sdks:java:extensions:sql:udf-test-provider:build")
+  dependsOn(":sdks:java:extensions:sql:udf:build")
+  dependsOn(":sdks:java:extensions:sql:zetasql:build")
 }
 
 tasks.register("javaPreCommitPortabilityApi") {
@@ -724,7 +722,6 @@ if (project.hasProperty("testJavaVersion")) {
 
   tasks.getByName("javaPreCommitPortabilityApi").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer")
   tasks.getByName("javaExamplesDataflowPrecommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer")
-  tasks.getByName("sqlPreCommit").dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer")
 } else {
   allprojects {
     tasks.withType(Test::class).configureEach {
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 359aeea55a2f..9576dd2c2f44 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -598,16 +598,16 @@ class BeamModulePlugin implements Plugin<Project> {
     def dbcp2_version = "2.9.0"
     def errorprone_version = "2.10.0"
     // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom
-    def gax_version = "2.36.0"
+    def gax_version = "2.38.0"
     def google_ads_version = "26.0.0"
     def google_clients_version = "2.0.0"
     def google_cloud_bigdataoss_version = "2.2.16"
     // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom
-    def google_cloud_spanner_version = "6.52.1"
+    def google_cloud_spanner_version = "6.55.0"
     def google_code_gson_version = "2.10.1"
     def google_oauth_clients_version = "1.34.1"
     // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom
-    def grpc_version = "1.58.0"
+    def grpc_version = "1.59.1"
     def guava_version = "32.1.2-jre"
     def hadoop_version = "2.10.2"
     def hamcrest_version = "2.1"
@@ -626,7 +626,7 @@ class BeamModulePlugin implements Plugin<Project> {
     def postgres_version = "42.2.16"
     def powermock_version = "2.0.9"
     // Try to keep protobuf_version consistent with the protobuf version in google_cloud_platform_libraries_bom
-    def protobuf_version = "3.24.4"
+    def protobuf_version = "3.25.1"
     def qpid_jms_client_version = "0.61.0"
     def quickcheck_version = "1.0"
     def sbe_tool_version = "1.25.1"
@@ -731,10 +731,10 @@ class BeamModulePlugin implements Plugin<Project> {
         // Keep version consistent with the version in google_cloud_resourcemanager, managed by google_cloud_platform_libraries_bom
         google_api_services_cloudresourcemanager    : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20230806-$google_clients_version",
         google_api_services_dataflow                : "com.google.apis:google-api-services-dataflow:v1b3-rev20220920-$google_clients_version",
-        google_api_services_healthcare              : "com.google.apis:google-api-services-healthcare:v1-rev20231003-$google_clients_version",
+        google_api_services_healthcare              : "com.google.apis:google-api-services-healthcare:v1-rev20231101-$google_clients_version",
         google_api_services_pubsub                  : "com.google.apis:google-api-services-pubsub:v1-rev20220904-$google_clients_version",
         // Keep version consistent with the version in google_cloud_nio, managed by google_cloud_platform_libraries_bom
-        google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20230907-$google_clients_version",
+        google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20231202-$google_clients_version",
         google_auth_library_credentials             : "com.google.auth:google-auth-library-credentials", // google_cloud_platform_libraries_bom sets version
         google_auth_library_oauth2_http             : "com.google.auth:google-auth-library-oauth2-http", // google_cloud_platform_libraries_bom sets version
         google_cloud_bigquery                       : "com.google.cloud:google-cloud-bigquery", // google_cloud_platform_libraries_bom sets version
@@ -754,7 +754,7 @@ class BeamModulePlugin implements Plugin<Project> {
         // The release notes shows the versions set by the BOM:
         // https://github.com/googleapis/java-cloud-bom/releases/tag/v26.21.0
         // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml
-        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.26.0",
+        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.28.0",
         google_cloud_spanner                        : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version
         google_cloud_spanner_test                   : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests",
         google_code_gson                            : "com.google.code.gson:gson:$google_code_gson_version",
@@ -2205,7 +2205,7 @@ class BeamModulePlugin implements Plugin<Project> {
       def goRootDir = "${project.rootDir}/sdks/go"
 
       // This sets the whole project Go version.
-      project.ext.goVersion = "go1.21.4"
+      project.ext.goVersion = "go1.21.5"
 
       // Minor TODO: Figure out if we can pull out the GOCMD env variable after goPrepare script
       // completion, and avoid this GOBIN substitution.
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy
index ddb317b89294..4b8a059109c9 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/GrpcVendoring_1_54_0.groovy
@@ -59,6 +59,7 @@ class GrpcVendoring_1_54_0 {
       "io.grpc:grpc-netty:$grpc_version",
       "io.grpc:grpc-protobuf:$grpc_version",
       "io.grpc:grpc-stub:$grpc_version",
+      "io.grpc:grpc-alts:$grpc_version",
       "io.grpc:grpc-testing:$grpc_version",
       // Use a classifier to ensure we get the jar containing native libraries. In the future
       // hopefully netty releases a single jar containing native libraries for all architectures.
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/kafka/KafkaTestUtilities.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/kafka/KafkaTestUtilities.groovy
index cd2875fdb512..bb08e79edd3c 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/kafka/KafkaTestUtilities.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/kafka/KafkaTestUtilities.groovy
@@ -40,7 +40,7 @@ class KafkaTestUtilities {
         '"keySizeBytes": "10",' +
         '"valueSizeBytes": "90"' +
         '}',
-        "--readTimeout=120",
+        "--readTimeout=60",
         "--kafkaTopic=beam",
         "--withTestcontainers=true",
         "--kafkaContainerVersion=5.5.2",
@@ -56,6 +56,7 @@ class KafkaTestUtilities {
           excludeTestsMatching "*SDFResumesCorrectly" //Kafka SDF does not work for kafka versions <2.0.1
           excludeTestsMatching "*StopReadingFunction" //Kafka SDF does not work for kafka versions <2.0.1
           excludeTestsMatching "*WatermarkUpdateWithSparseMessages" //Kafka SDF does not work for kafka versions <2.0.1
+          excludeTestsMatching "*KafkaIOSDFReadWithErrorHandler"
         }
       }
     }
diff --git a/contributor-docs/rc-testing-guide.md b/contributor-docs/rc-testing-guide.md
new file mode 100644
index 000000000000..0ac932711206
--- /dev/null
+++ b/contributor-docs/rc-testing-guide.md
@@ -0,0 +1,60 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Release Candidate (RC) Testing Guide
+
+This guide is meant for anybody who is interested in testing Beam Release Candidates against downstream projects. Note
+that one need not have any status on the Apache Beam project (eg. PMC Member, Committer) to vote; all are welcome.
+Please subscribe to the [dev list](https://lists.apache.org/list.html?dev@beam.apache.org), and vote on the RC Vote email thread.
+
+
+## RC Testing Objectives
+
+The RC testing process aims to:
+
+ - Test new release candidates against existing code bases utilizing Apache Beam, to ensure there are no unexpected behaviors downstream.
+ - Incorporate a breadth of perspectives (including validation on multiple SDKs and multiple runners), before releasing a new version.
+ - Allow Beam Contributors to dogfood their changes and verify that they work as intended.
+
+
+## Beam Release process overview
+- For a comprehensive overview on the Beam release process, please take a look at our [release guide](https://github.com/apache/beam/blob/master/contributor-docs/release-guide.md).
+- Note that release candidate votes will be open for 72 hours after the voting email is sent.
+
+
+## Ideas for Python SDK Validators
+
+_Note: Do the following in a dev-like environment._
+- If you are a Python SDK user that utilizes notebooks (eg. Jupyter Notebooks, or Colab Notebooks), change `pip install`
+to point to the new RC (e.g. `pip install apache_beam[gcp]==2.52.0rc1`). Re-execute the workflow to ensure everything
+works as intended.
+- If your workflow utilizes [Dataflow Templates](https://github.com/GoogleCloudPlatform/DataflowTemplates), or another way of launching your job, modify your `requirements.txt` file, `setup.py` file, or `DockerFile` to point to the new Beam RC.
+- _Tip_: Run your pipeline both against Direct Runner, and another runner of your choice by modifying your job's `PipelineOptions`.
+
+
+## Ideas for Java SDK Validators
+_Note: Do the following in a dev-like environment._
+- If you are a Java SDK user that utilizes Maven or Gradle in your workflow, in your `pom.xml` file, modify the `beam.version` to reflect the newest RC, and modify the `<url>` tag under `<repository>`, to point to the repository given in the vote email. [Example Changes](https://github.com/GoogleCloudPlatform/DataflowTemplates/pull/1090/files).
+
+
+## Ideas for Go SDK Validators
+_Note: Do the following in a dev-like environment._
+- If you utilize the Go SDK, use `go-get` to use the desired RC, for example, `go get -d github.com/apache/beam/sdks/v2@v2.xx.0-RC1`
+- Utilize the `--environment_config`, to point to the new release, for example, `--environment_config=apache/beam_go_sdk:2.xx.0rc1`
+
+
+## After validation
+
+- Reply to [dev list](https://lists.apache.org/list.html?dev@beam.apache.org) vote thread with your [vote](https://www.apache.org/foundation/voting.html) and an explanation of the use case you tested.
+- [Optional]: If your use case can be well represented by a test, consider contributing a test to Beam! However, note that the of the value of validation is manual testing outside of the Beam CI workflow, so it is still recommended (and highly encouraged!) to validate these test cases in future releases.
\ No newline at end of file
diff --git a/contributor-docs/release-guide.md b/contributor-docs/release-guide.md
index faa8ad5927cf..b3b3213d9c97 100644
--- a/contributor-docs/release-guide.md
+++ b/contributor-docs/release-guide.md
@@ -271,12 +271,6 @@ fixes. This is because bugs are always being introduced as part of normal
 development. We cut the branch to prevent new bugs being introduced and _then_
 we fix and cherrypick any truly release-blocking problems.
 
-In order to run this workflow, you will need to provide a Apache ID and Jenkins
-API token.  Your Jenkins API token can be generated by visiting
-https://ci-beam.apache.org, signing in with your Apache credentials, then going
-to `https://ci-beam.apache.org/user/<your username>/configure` and clicking
-`Add new token` in the API token section.
-
 - [ ] Run
   [cut_release_branch](https://github.com/apache/beam/actions/workflows/cut_release_branch.yml)
   (click `run workflow`)
@@ -288,18 +282,14 @@ diagram:
 
 This should be accomplished by the
 [cut_release_branch](https://github.com/apache/beam/actions/workflows/cut_release_branch.yml)
-workflow. This workflow will also update
-[mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py)
-to contain all of the active Jenkins jobs.
+workflow.
 
 The following must be manually done or confirmed:
 
 - [ ] The `master` branch has the SNAPSHOT/dev version incremented.
 - [ ] The release branch has the SNAPSHOT/dev version to be released.
 - [ ] The Dataflow container image should be modified to the version to be released.
-- [ ] Due to a bug/limitation in the workflow, you must navigate to the pull
-      request found in the logs and comment `Run Gradle Publish`.
-- [ ] After publish, close the PR.
+- [ ] Due to current limitation in the workflow, you must navigate to https://github.com/apache/beam/actions/workflows/beam_Release_NightlySnapshot.yml and click "Run workflow" and select the branch just created (release-2.xx.0) to build a snapshot.
 - [ ] Manually update `CHANGES.md` on `master` by adding a new section for the
   next release
   ([example](https://github.com/apache/beam/commit/96ab1fb3fe07acf7f7dc9d8c829ae36890d1535c)).
@@ -357,13 +347,9 @@ issues that would block the creation of the release candidate.
        ```
        (cd release/src/main/scripts && ./verify_release_build.sh)
        ```
-    4. Trigger all Github Action and Jenkins PostCommit jobs from the PR created by the previous step.
+    4. Trigger all Github Action jobs from the PR created by the previous step.
        For GitHub Action jobs, they should be triggered by the pull_request_target event of a specific placeholder file
        added to the PR (`release/trigger_all_tests.json`), so no additional action should be needed.
-       You can run [mass_comment.py](https://github.com/apache/beam/blob/master/release/src/main/scripts/mass_comment.py) to do that.
-       Or manually add one trigger phrase per PR comment for Jenkins tests, or rerun the workflow for GitHub Action tests.
-       See [jenkins_jobs.txt](https://github.com/apache/beam/blob/master/release/src/main/scripts/jenkins_jobs.txt)
-       for a full list of phrases.
 
 - **Tasks included in the script**
    - Installs `hub` with your agreement and setup local git repo;
@@ -817,7 +803,7 @@ template; please adjust as you see fit.
     Reviewers are encouraged to test their own use cases with the release candidate, and vote +1 if
     no issues are found. Only PMC member votes will count towards the final vote, but votes from all
     community members is encouraged and helpful for finding regressions; you can either test your own
-    use cases or use cases from the validation sheet [10].
+    use cases [13] or use cases from the validation sheet [10].
 
     The complete staging area is available for your review, which includes:
     * GitHub Release notes [1],
@@ -833,7 +819,7 @@ template; please adjust as you see fit.
 
     The vote will be open for at least 72 hours. It is adopted by majority approval, with at least 3 PMC affirmative votes.
 
-    For guidelines on how to try the release in your projects, check out our blog post at https://beam.apache.org/blog/validate-beam-release/.
+    For guidelines on how to try the release in your projects, check out our RC testing guide [13].
 
     Thanks,
     Release Manager
@@ -850,6 +836,7 @@ template; please adjust as you see fit.
     [10] https://docs.google.com/spreadsheets/d/1qk-N5vjXvbcEk68GjbkSZTR8AGqyNUM-oLFo_ZXBpJw/edit#gid=...
     [11] https://hub.docker.com/search?q=apache%2Fbeam&type=image
     [12] https://github.com/apache/beam/pull/...
+    [13] https://github.com/apache/beam/blob/master/contributor-docs/rc-testing-guide.md
 
 If there are any issues found in the release candidate, reply on the vote
 thread to cancel the vote.  There’s no need to wait 72 hours. Go back to
@@ -860,7 +847,8 @@ pull request, just correct it on the spot and the vote can continue as-is.
 ### Run validation tests
 
 The community is responsible for performing validation, but as release manager
-you are expected to contribute as well.
+you are expected to contribute as well. Please see the [RC Testing Guide](https://github.com/apache/beam/blob/master/contributor-docs/rc-testing-guide.md)
+for ideas on helping validate testing on downstream projects.
 
 Before accepting an RC, as a community we try to exercise most (if not all) of
 the tests listed in this
@@ -1229,13 +1217,16 @@ Ping [dev@](mailto:dev@beam.apache.org) mailing list for assistance if you need
 Copy the source release from the `dev` repository to the `release` repository at `dist.apache.org` using Subversion.
 
 ```
+export RELEASE_VERSION=<set the release version>
+export OLD_RELEASE_VERSION=<set the previous release version>
+
 svn co https://dist.apache.org/repos/dist/dev/beam dev  # Checkout the `dev` artifact repo.
 
 svn co https://dist.apache.org/repos/dist/release/beam release  # Checkout the `release` artifact repo.
 
 mkdir release/$RELEASE_VERSION
 
-# Copy files from the `dev` artifact repo to the `release` artifact repo.
+cp -R dev/$RELEASE_VERSION/* release/$RELEASE_VERSION/
 
 cd release
 
@@ -1338,6 +1329,18 @@ Release Manager
 [1] https://github.com/apache/beam/pull/123
 ```
 
+### Update the Java starter repo
+
+After the new Beam release is published, the Java starter project needs to have its version manually upgraded.
+To do this, create a PR like https://github.com/apache/beam-starter-java/pull/94 (with the appropriate version
+number).
+
+#### (Optional) Update the remaining starter repos
+
+You can also update the versions in https://github.com/apache/beam-starter-python and
+https://github.com/apache/beam-starter-go if you would like. This is optional because dependabot will automatically
+open a PR to do this if you don't.
+
 ### Update Beam Playground
 
 After new Beam Release is published, Beam Playground can be updated following the steps below:
diff --git a/examples/java/src/main/java/org/apache/beam/examples/KafkaStreaming.java b/examples/java/src/main/java/org/apache/beam/examples/KafkaStreaming.java
index 34a4b646555d..5bad7911e646 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/KafkaStreaming.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/KafkaStreaming.java
@@ -49,8 +49,11 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.Combine;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.Sum;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
 import org.apache.beam.sdk.transforms.windowing.AfterProcessingTime;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -60,6 +63,8 @@
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.kafka.common.errors.SerializationException;
+import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.IntegerDeserializer;
 import org.apache.kafka.common.serialization.IntegerSerializer;
 import org.apache.kafka.common.serialization.StringDeserializer;
@@ -208,15 +213,22 @@ public void run() {
       // Start reading form Kafka with the latest offset
       consumerConfig.put("auto.offset.reset", "latest");
 
-      PCollection<KV<String, Integer>> pCollection =
-          pipeline.apply(
-              KafkaIO.<String, Integer>read()
-                  .withBootstrapServers(options.getKafkaHost())
-                  .withTopic(TOPIC_NAME)
-                  .withKeyDeserializer(StringDeserializer.class)
-                  .withValueDeserializer(IntegerDeserializer.class)
-                  .withConsumerConfigUpdates(consumerConfig)
-                  .withoutMetadata());
+      // Register an error handler for any deserialization errors.
+      // Errors are simulated with an intentionally failing deserializer
+      PCollection<KV<String, Integer>> pCollection;
+      try (BadRecordErrorHandler<PCollection<BadRecord>> errorHandler =
+          pipeline.registerBadRecordErrorHandler(new LogErrors())) {
+        pCollection =
+            pipeline.apply(
+                KafkaIO.<String, Integer>read()
+                    .withBootstrapServers(options.getKafkaHost())
+                    .withTopic(TOPIC_NAME)
+                    .withKeyDeserializer(StringDeserializer.class)
+                    .withValueDeserializer(IntermittentlyFailingIntegerDeserializer.class)
+                    .withConsumerConfigUpdates(consumerConfig)
+                    .withBadRecordErrorHandler(errorHandler)
+                    .withoutMetadata());
+      }
 
       pCollection
           // Apply a window and a trigger ourput repeatedly.
@@ -317,4 +329,39 @@ public void processElement(ProcessContext c, IntervalWindow w) throws Exception
       c.output(c.element());
     }
   }
+
+  // Simple PTransform to log Error information
+  static class LogErrors extends PTransform<PCollection<BadRecord>, PCollection<BadRecord>> {
+
+    @Override
+    public PCollection<BadRecord> expand(PCollection<BadRecord> input) {
+      return input.apply("Log Errors", ParDo.of(new LogErrorFn()));
+    }
+
+    static class LogErrorFn extends DoFn<BadRecord, BadRecord> {
+      @ProcessElement
+      public void processElement(@Element BadRecord record, OutputReceiver<BadRecord> receiver) {
+        System.out.println(record);
+        receiver.output(record);
+      }
+    }
+  }
+
+  // Intentionally failing deserializer to simulate bad data from Kafka
+  public static class IntermittentlyFailingIntegerDeserializer implements Deserializer<Integer> {
+
+    public static final IntegerDeserializer INTEGER_DESERIALIZER = new IntegerDeserializer();
+    public int deserializeCount = 0;
+
+    public IntermittentlyFailingIntegerDeserializer() {}
+
+    @Override
+    public Integer deserialize(String topic, byte[] data) {
+      deserializeCount++;
+      if (deserializeCount % 10 == 0) {
+        throw new SerializationException("Expected Serialization Exception");
+      }
+      return INTEGER_DESERIALIZER.deserialize(topic, data);
+    }
+  }
 }
diff --git a/examples/notebooks/beam-ml/README.md b/examples/notebooks/beam-ml/README.md
index 0ae937e9e284..f1c19747fc71 100644
--- a/examples/notebooks/beam-ml/README.md
+++ b/examples/notebooks/beam-ml/README.md
@@ -16,7 +16,7 @@
     specific language governing permissions and limitations
     under the License.
 -->
-# ML Sample Notebooks
+# ML sample notebooks
 
 Starting with the Apache Beam SDK version 2.40, users have access to a
 [RunInference](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.RunInference)
@@ -25,15 +25,15 @@ transform.
 This transform allows you to make predictions and inference on data with machine learning (ML) models.
 The model handler abstracts the user from the configuration needed for
 specific frameworks, such as Tensorflow, PyTorch, and others. For a full list of supported frameworks,
-see the Apache Beam [Machine Learning](https://beam.apache.org/documentation/sdks/python-machine-learning) page.
+see the [About Beam ML](https://beam.apache.org/documentation/ml/about-ml/) page.
 
-## Using The Notebooks
+## Use the notebooks
 
 These notebooks illustrate ways to use Apache Beam's RunInference transforms, as well as different
-use cases for [ModelHandler](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.ModelHandler) implementations.
-Beam comes with [multiple ModelHandler implementations](https://beam.apache.org/documentation/sdks/python-machine-learning/#modify-a-pipeline-to-use-an-ml-model).
+use cases for [`ModelHandler`](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.ModelHandler) implementations.
+Beam comes with [multiple `ModelHandler` implementations](https://beam.apache.org/documentation/ml/about-ml/#modify-a-python-pipeline-to-use-an-ml-model).
 
-### Loading the Notebooks
+### Load the notebooks
 
 1. To get started quickly with notebooks, use [Colab](https://colab.sandbox.google.com/).
 2. In Colab, open the notebook from GitHub using the notebook URL, for example:
@@ -48,6 +48,14 @@ to your project and bucket.
 
 This section contains the following example notebooks.
 
+### Data processing
+
+* [Generate text embeddings by using the Vertex AI API](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb)
+* [Generate text embeddings by using Hugging Face Hub models](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb)
+* [Compute and apply vocabulary on a dataset](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/compute_and_apply_vocab.ipynb)
+* [Use MLTransform to scale data](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/scale_data.ipynb)
+* [Preprocessing with the Apache Beam DataFrames API](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/dataframe_api_preprocessing.ipynb)
+
 ### Prediction and inference with pretrained models
 
 * [Apache Beam RunInference for PyTorch](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_pytorch.ipynb)
@@ -85,8 +93,3 @@ This section contains the following example notebooks.
 ### Model Evaluation
 
 * [Use TFMA to evaluate and compare model performance](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/tfma_beam.ipynb)
-
-### Data processing
-
-* [Preprocess data with MLTransform](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/mltransform_basic.ipynb)
-* [Preprocessing with the Apache Beam DataFrames API](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/dataframe_api_preprocessing.ipynb)
diff --git a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb
index cf05979c5b33..b0e9bc2f53e4 100644
--- a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb
+++ b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb
@@ -15,16 +15,6 @@
     }
   },
   "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "view-in-github",
-        "colab_type": "text"
-      },
-      "source": [
-        "<a href=\"https://colab.research.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/automatic_model_refresh.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
-      ]
-    },
     {
       "cell_type": "code",
       "source": [
diff --git a/examples/notebooks/beam-ml/data_preprocessing/compute_and_apply_vocab.ipynb b/examples/notebooks/beam-ml/data_preprocessing/compute_and_apply_vocab.ipynb
new file mode 100644
index 000000000000..ee47cb7711fa
--- /dev/null
+++ b/examples/notebooks/beam-ml/data_preprocessing/compute_and_apply_vocab.ipynb
@@ -0,0 +1,419 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "source": [
+        "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n",
+        "\n",
+        "# Licensed to the Apache Software Foundation (ASF) under one\n",
+        "# or more contributor license agreements. See the NOTICE file\n",
+        "# distributed with this work for additional information\n",
+        "# regarding copyright ownership. The ASF licenses this file\n",
+        "# to you under the Apache License, Version 2.0 (the\n",
+        "# \"License\"); you may not use this file except in compliance\n",
+        "# with the License. You may obtain a copy of the License at\n",
+        "#\n",
+        "#   http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing,\n",
+        "# software distributed under the License is distributed on an\n",
+        "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n",
+        "# KIND, either express or implied. See the License for the\n",
+        "# specific language governing permissions and limitations\n",
+        "# under the License"
+      ],
+      "metadata": {
+        "id": "n2PtbQM61lFZ"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Compute and apply vocabulary on a dataset\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/compute_and_apply_vocab.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/colab_32px.png\" />Run in Google Colab</a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/compute_and_apply_vocab.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/github_32px.png\" />View source on GitHub</a>\n",
+        "  </td>\n",
+        "</table>\n"
+      ],
+      "metadata": {
+        "id": "ZUSiAR62SgO8"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "The [`ComputeAndApplyVocabulary`](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ComputeAndApplyVocabulary) data processing transform computes a unique vocabulary from a dataset and then maps each word or token to a distinct integer index. Use this transform to change textual data into numerical representations for machine learning (ML) tasks.\n",
+        "\n",
+        "When you train ML models that use text data, generating a vocabulary on the incoming dataset is an important preprocessing step. By mapping words to numerical indices, the vocabulary reduces the complexity and dimensionality of dataset. This step allows ML models to process the same words in a consistent way.\n",
+        "\n",
+        "This notebook shows how to use `MLTransform` to complete the following tasks:\n",
+        "* Use `write` mode to generate a vocabulary on the input text and assign an index value to each token.\n",
+        "* Use `read` mode to use the generated vocabulary and assign an index to a different dataset.\n",
+        "\n",
+        "`MLTransform` uses the `ComputeAndApplyVocabulary` transform, which is implemented by using `tensorflow_transform` to generate the vocabulary.\n",
+        "\n",
+        "For more information about using `MLTransform`, see [Preprocess data with MLTransform](https://beam.apache.org/documentation/ml/preprocess-data/) in the Apache Beam documentation"
+      ],
+      "metadata": {
+        "id": "Ebs9JtTrqQZH"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Install the required modules\n",
+        "\n",
+        "To use `ComputeAndVocabulary` with `MLTransfrom`, install `tensorflow_transform` and the Apache Beam SDK version 2.53.0 or later."
+      ],
+      "metadata": {
+        "id": "BnOMWq9C1Yo0"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "P3uUPotJo7Dc"
+      },
+      "outputs": [],
+      "source": [
+        "! pip install apache_beam>=2.53.0 --quiet\n",
+        "! pip install tensorflow-transform --quiet"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "import tempfile\n",
+        "import apache_beam as beam\n",
+        "from apache_beam.ml.transforms.base import MLTransform\n",
+        "from apache_beam.ml.transforms.tft import ComputeAndApplyVocabulary"
+      ],
+      "metadata": {
+        "id": "e7-Vmawtq3xC"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Use the artifact location\n",
+        "\n",
+        "In `write` mode, the artifact location is used to store artifacts, such as the vocabulary file generated by `ComputeAndApplyVocabulary`.\n",
+        "\n",
+        "**NOTE**: The artifact location must be empty. If it isn't empty, a `RuntimeError` occurs.\n",
+        "\n",
+        "In `read` mode, `MLTransform` fetches artifacts from the specified artifact location. Pass the same artifact location that you used in `write` mode. Otherwise, a `RuntimeError` occurs or `MLTransform` produces unexpected results in `read` mode.\n"
+      ],
+      "metadata": {
+        "id": "vfarBxAMFvRA"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artifact_location = tempfile.mkdtemp(prefix='compute_and_apply_vocab_')\n",
+        "artifact_location_with_frequency_threshold = tempfile.mkdtemp(prefix='compute_and_apply_vocab_frequency_threshold_')"
+      ],
+      "metadata": {
+        "id": "wkVNu3SJrYnc"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "documents = [\n",
+        "    {\"feature\": \"the quick brown fox jumps over the lazy dog\"},\n",
+        "    {\"feature\": \"the five boxing wizards jump quickly in the sky\"},\n",
+        "    {\"feature\": \"dogs are running in the park\"},\n",
+        "    {\"feature\": \"the quick brown fox\"}\n",
+        "]"
+      ],
+      "metadata": {
+        "id": "s3bYkvXlrnRt"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "In this example, in `write` mode, `MLTransform` uses `ComputeAndApplyVocabulary` to generate vocabulary on the incoming dataset. The incoming text data is split into tokens. Each token is assigned an unique index.\n",
+        "\n",
+        " The generated vocabulary is stored in an artifact location that you can use on a different dataset in `read` mode."
+      ],
+      "metadata": {
+        "id": "oETBJNVfRws_"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(documents)\n",
+        "  # Compute and apply vocabulary by using MLTransform.\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location).with_transform(\n",
+        "          ComputeAndApplyVocabulary(columns=['feature'], split_string_by_delimiter=' ', vocab_filename='vocab_index'))\n",
+        "      )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "6hAYN0ZYt2da",
+        "outputId": "2b843ab1-9b72-4f31-d9b9-58c2e0c0fe75"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(feature=array([ 0,  1,  4,  3, 12, 10,  0, 11, 16]))\n",
+            "Row(feature=array([ 0, 14, 17,  5, 13,  8,  2,  0,  6]))\n",
+            "Row(feature=array([15, 18,  7,  2,  0,  9]))\n",
+            "Row(feature=array([0, 1, 4, 3]))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Understand and visualize vocabulary\n",
+        "\n",
+        "When working with text data in machine learning, one common step is the generation of a vocabulary index. `MLTransform` completes this step by using the `ComputeAndApplyVocabulary` transformation. Each unique word in your text data is assigned a specific index. This index is then used to represent the text in a numerical format, which is needed for machine learning algorithms.\n",
+        "\n",
+        "In this example, the `ComputeAndApplyVocabulary` transformation is applied to the `feature` column. A vocabulary index is created for each unique word found in this column.\n",
+        "\n",
+        "To visualize and understand this generated vocabulary, use the `ArtifactsFetcher` class. This class allows you to retrieve the vocabulary list from your specified location. When you have this list, you can see the index associated with each word in your vocabulary. This index corresponds to the numerical representation used in the transformation output of `ComputeAndApplyVocabulary`.\n",
+        "\n",
+        "Examine this vocabulary index to understand how your text data is being processed and represented numerically. This understanding is useful for debugging and improving machine learning models that rely on text data."
+      ],
+      "metadata": {
+        "id": "hvTvzOw8iBi9"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from apache_beam.ml.transforms.utils import ArtifactsFetcher\n",
+        "artifact_fetcher = ArtifactsFetcher(artifact_location)\n",
+        "vocab_list = artifact_fetcher.get_vocab_list(vocab_filename='vocab_index_feature')\n",
+        "for i in range(len(vocab_list)):\n",
+        "  print(f'{i}: {vocab_list[i]}')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "dzmd_4GaiACA",
+        "outputId": "d73a5bee-15bc-4cb4-d2e0-59398a62cd60"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "0: the\n",
+            "1: quick\n",
+            "2: in\n",
+            "3: fox\n",
+            "4: brown\n",
+            "5: wizards\n",
+            "6: sky\n",
+            "7: running\n",
+            "8: quickly\n",
+            "9: park\n",
+            "10: over\n",
+            "11: lazy\n",
+            "12: jumps\n",
+            "13: jump\n",
+            "14: five\n",
+            "15: dogs\n",
+            "16: dog\n",
+            "17: boxing\n",
+            "18: are\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Set the frequency threshold\n",
+        "\n",
+        "The `frequency_threshold` parameter identifies the elements that appear frequently in the dataset. This parameter limits the generated vocabulary to elements with an absolute frequency greater than or equal to the specified threshold. If you don't specify the parameter, the entire vocabulary is generated.\n",
+        "\n",
+        "If the frequency of a vocabulary item is less than the threshold, it's assigned a default value. You can use the `default_value` parameter to set this value. Otherwise, it defaults to `-1`."
+      ],
+      "metadata": {
+        "id": "oZZtAVk4wJCd"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(documents)\n",
+        "  # Compute and apply vocabulary by using MLTransform.\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location_with_frequency_threshold).with_transform(\n",
+        "          ComputeAndApplyVocabulary(columns=['feature'], split_string_by_delimiter=' ', frequency_threshold=2, vocab_filename='vocab_index'))\n",
+        "      )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "FpDkOSWtv1pn",
+        "outputId": "ff74a769-534c-4f40-fc59-bf4a10eaea7c"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(feature=array([ 0,  1,  4,  3, -1, -1,  0, -1, -1]))\n",
+            "Row(feature=array([ 0, -1, -1, -1, -1, -1,  2,  0, -1]))\n",
+            "Row(feature=array([-1, -1, -1,  2,  0, -1]))\n",
+            "Row(feature=array([0, 1, 4, 3]))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "In the output, if the frequency of the token is less than the specified frequency, it's assigned to a `default_value` of `-1`. For the other tokens, a vocabulary file is generated."
+      ],
+      "metadata": {
+        "id": "h1s4a6hzxKrb"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from apache_beam.ml.transforms.utils import ArtifactsFetcher\n",
+        "artifact_fetcher = ArtifactsFetcher(artifact_location_with_frequency_threshold)\n",
+        "vocab_list = artifact_fetcher.get_vocab_list(vocab_filename='vocab_index_feature')\n",
+        "for i in range(len(vocab_list)):\n",
+        "  print(f'{i}: {vocab_list[i]}')"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "rwgthpj9x8kp",
+        "outputId": "e5fcdd51-75c1-4da1-d5a4-d7f39129cc85"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "0: the\n",
+            "1: quick\n",
+            "2: in\n",
+            "3: fox\n",
+            "4: brown\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Use MLTransform for inference workloads\n",
+        "\n",
+        "When `MLTransform` is in `write` mode, it produces artifacts, such as vocabulary files for `ComputeAndApplyVocabulary`. These artifacts allow you to apply the same vocabulary, and any other preprocessing transforms, when you train your model and serve it in production, or when you test its accuracy.\n",
+        "\n",
+        "When `MLTransform` is used `read` mode, it uses the previously generated vocabulary files to map the incoming text data. If the incoming vocabulary isn't found in the generated vocabulary, then the incoming vocabulary is mapped to a `default_value` provided during `write` mode. In this case, the `default_value` is `-1`.\n",
+        "\n",
+        "When `MLTransform` is in `write` mode, it produces artifacts, such as vocabulary files for `ComputeAndApplyVocabulary`.\n",
+        "\n"
+      ],
+      "metadata": {
+        "id": "PEqYpCexybO4"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "test_documents = [\n",
+        "    {'feature': 'wizards are flying in the sky'},\n",
+        "    {'feature': 'I love dogs'}\n",
+        "]\n",
+        "\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(test_documents)\n",
+        "  # Compute and apply vocabulary by using MLTransform.\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(read_artifact_location=artifact_location))\n",
+        "\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "x-LRXUaTyX_B",
+        "outputId": "0e0b04df-a684-49c9-fe72-a032c4726078"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(feature=array([ 5, 18, -1,  2,  0,  6]))\n",
+            "Row(feature=array([-1, -1, 15]))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "When you specify `read_artifact_location`, you don't have to pass any transforms to `MLTransform`. Instead, `MLTransform` saves the artifacts and the transforms produced in the location specified by `write_artifact_location`."
+      ],
+      "metadata": {
+        "id": "qwqH283F0kee"
+      }
+    }
+  ]
+}
diff --git a/examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb b/examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb
new file mode 100644
index 000000000000..30bbadf5b9b0
--- /dev/null
+++ b/examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb
@@ -0,0 +1,399 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": 27,
+      "metadata": {
+        "id": "UmEFwsNs1OES"
+      },
+      "outputs": [],
+      "source": [
+        "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n",
+        "\n",
+        "# Licensed to the Apache Software Foundation (ASF) under one\n",
+        "# or more contributor license agreements. See the NOTICE file\n",
+        "# distributed with this work for additional information\n",
+        "# regarding copyright ownership. The ASF licenses this file\n",
+        "# to you under the Apache License, Version 2.0 (the\n",
+        "# \"License\"); you may not use this file except in compliance\n",
+        "# with the License. You may obtain a copy of the License at\n",
+        "#\n",
+        "#   http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing,\n",
+        "# software distributed under the License is distributed on an\n",
+        "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n",
+        "# KIND, either express or implied. See the License for the\n",
+        "# specific language governing permissions and limitations\n",
+        "# under the License"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Generate text embeddings by using Hugging Face Hub models\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/colab_32px.png\" />Run in Google Colab</a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/huggingface_text_embeddings.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/github_32px.png\" />View source on GitHub</a>\n",
+        "  </td>\n",
+        "</table>\n"
+      ],
+      "metadata": {
+        "id": "ZUSiAR62SgO8"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "\n",
+        "Use text embeddings to represent text as numerical vectors. This process lets computers understand and process text data, which is essential for many natural language processing (NLP) tasks.\n",
+        "\n",
+        "The following NLP tasks use embeddings:\n",
+        "\n",
+        "* **Semantic search:** Find documents or passages that are relevant to a query when the query doesn't use the exact same words as the documents.\n",
+        "* **Text classification:** Categorize text data into different classes, such as spam and not spam, or positive sentiment and negative sentiment.\n",
+        "* **Machine translation:** Translate text from one language to another and preserve the meaning.\n",
+        "* **Text summarization:** Create shorter summaries of text.\n",
+        "\n",
+        "This notebook uses Apache Beam's `MLTransform` to generate embeddings from text data.\n",
+        "\n",
+        "Hugging Face's [`SentenceTransformers`](https://huggingface.co/sentence-transformers) framework uses Python to generate sentence, text, and image embeddings.\n",
+        "\n",
+        "To generate text embeddings that use Hugging Face models and `MLTransform`, use the `SentenceTransformerEmbeddings` module to specify the model configuration.\n"
+      ],
+      "metadata": {
+        "id": "yvVIEhF01ZWq"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Install dependencies\n",
+        "\n",
+        "Install Apache Beam and the dependencies needed to work with Hugging Face embeddings. The dependencies includes the `sentence-transformers` package, which is required to use the `SentenceTransformerEmbeddings` module.\n"
+      ],
+      "metadata": {
+        "id": "jqYXaBJ821Zs"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "! pip install apache_beam>=2.53.0 --quiet\n",
+        "! pip install sentence-transformers --quiet"
+      ],
+      "metadata": {
+        "id": "shzCUrZI1XhF"
+      },
+      "execution_count": 28,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import tempfile\n",
+        "import apache_beam as beam\n",
+        "from apache_beam.ml.transforms.base import MLTransform\n",
+        "from apache_beam.ml.transforms.embeddings.huggingface import SentenceTransformerEmbeddings"
+      ],
+      "metadata": {
+        "id": "jVxSi2jS3M3b"
+      },
+      "execution_count": 29,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Process the data\n",
+        "\n",
+        "`MLTransform` is a `PTransform` that you can use for data preparation, including generating text embeddings.\n",
+        "\n",
+        "### Use MLTransform in write mode\n",
+        "\n",
+        "In `write` mode, `MLTransform` saves the transforms and their attributes to an artifact location. Then, when you run `MLTransform` in `read` mode, these transforms are used. This process ensures that you're applying the same preprocessing steps when you train your model and when you serve the model in production or test its accuracy.\n",
+        "\n",
+        "For more information about using `MLTransform`, see [Preprocess data with MLTransform](https://beam.apache.org/documentation/ml/preprocess-data/) in the Apache Beam documentation."
+      ],
+      "metadata": {
+        "id": "kXDM8C7d3nPV"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Get the data\n",
+        "\n",
+        "The following text inputs come from the Hugging Face blog [Getting Started With Embeddings](https://huggingface.co/blog/getting-started-with-embeddings).\n",
+        "\n",
+        "\n",
+        "`MLTransform` operates on dictionaries of data. To generate embeddings for specific columns, provide the column names as input to the `columns` argument in the `SentenceTransformerEmbeddings` package."
+      ],
+      "metadata": {
+        "id": "Dbkmu3HP6Kql"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "content = [\n",
+        "    {'x': 'How do I get a replacement Medicare card?'},\n",
+        "    {'x': 'What is the monthly premium for Medicare Part B?'},\n",
+        "    {'x': 'How do I terminate my Medicare Part B (medical insurance)?'},\n",
+        "    {'x': 'How do I sign up for Medicare?'},\n",
+        "    {'x': 'Can I sign up for Medicare Part B if I am working and have health insurance through an employer?'},\n",
+        "    {'x': 'How do I sign up for Medicare Part B if I already have Part A?'},\n",
+        "    {'x': 'What are Medicare late enrollment penalties?'},\n",
+        "    {'x': 'What is Medicare and who can get it?'},\n",
+        "    {'x': 'How can I get help with my Medicare Part A and Part B premiums?'},\n",
+        "    {'x': 'What are the different parts of Medicare?'},\n",
+        "    {'x': 'Will my Medicare premiums be higher because of my higher income?'},\n",
+        "    {'x': 'What is TRICARE ?'},\n",
+        "    {'x': \"Should I sign up for Medicare Part B if I have Veterans' Benefits?\"}\n",
+        "]\n",
+        "\n",
+        "text_embedding_model_name = 'sentence-transformers/sentence-t5-large'\n",
+        "\n",
+        "\n",
+        "# helper function that returns a dict containing only first\n",
+        "# ten elements of generated embeddings\n",
+        "def truncate_embeddings(d):\n",
+        "  for key in d.keys():\n",
+        "    d[key] = d[key][:10]\n",
+        "  return d"
+      ],
+      "metadata": {
+        "id": "LCTUs8F73iDg"
+      },
+      "execution_count": 30,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "\n",
+        "### Generate text embeddings\n",
+        "This example uses the model `sentence-transformers/sentence-t5-large` to generate text embeddings. The model uses only the encoder from a `T5-large model`. The weights are stored in FP16. For more information about the model, see [Sentence-T5: Scalable Sentence Encoders from Pre-trained Text-to-Text Models](https://arxiv.org/abs/2108.08877)."
+      ],
+      "metadata": {
+        "id": "SApMmlRLRv_e"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artifact_location_t5 = tempfile.mkdtemp(prefix='huggingface_')\n",
+        "embedding_transform = SentenceTransformerEmbeddings(\n",
+        "        model_name=text_embedding_model_name, columns=['x'])\n",
+        "\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = (\n",
+        "      pipeline\n",
+        "      | \"CreateData\" >> beam.Create(content))\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location_t5).with_transform(embedding_transform))\n",
+        "\n",
+        "  transformed_pcoll | beam.Map(truncate_embeddings) | 'LogOutput' >> beam.Map(print)\n",
+        "\n",
+        "  transformed_pcoll | \"PrintEmbeddingShape\" >> beam.Map(lambda x: print(f\"Embedding shape: {len(x['x'])}\"))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "SF6izkN134sf",
+        "outputId": "524d3506-d31f-4dee-9079-1ed6d7cadf1a"
+      },
+      "execution_count": 31,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'x': [-0.0317193828523159, -0.005265399813652039, -0.012499183416366577, 0.00018130357784684747, -0.005592408124357462, 0.06207558885216713, -0.01656288281083107, 0.0167048592120409, -0.01239298190921545, 0.03041897714138031]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.015295305289328098, 0.005405726842582226, -0.015631258487701416, 0.022797023877501488, -0.027843449264764786, 0.03968179598450661, -0.004387892782688141, 0.022909151390194893, 0.01015392318367958, 0.04723235219717026]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.03450256213545799, -0.002632762538269162, -0.022460950538516045, -0.011689935810863972, -0.027329981327056885, 0.07293087989091873, -0.03069353476166725, 0.05429817736148834, -0.01308195199817419, 0.017668722197413445]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.02869587577879429, -0.0002648509689606726, -0.007186499424278736, -0.0003750955802388489, 0.012458174489438534, 0.06721009314060211, -0.013404129073023796, 0.03204648941755295, -0.021021844819188118, 0.04968355968594551]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.03241290897130966, 0.006845517549663782, 0.02001815102994442, -0.0057969288900494576, 0.008191823959350586, 0.08160955458879471, -0.009215254336595535, 0.023534387350082397, -0.02034241147339344, 0.0357462577521801]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.04592451825737953, -0.0025395643897354603, -0.01178023498505354, 0.011568977497518063, -0.0029014083556830883, 0.06971456110477448, -0.021167151629924774, 0.015902182087302208, -0.015007994137704372, 0.026213033124804497]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [0.005221465136855841, -0.002127869985997677, -0.002369001042097807, -0.019337018951773643, 0.023243796080350876, 0.05599674955010414, -0.022721167653799057, 0.024813007563352585, -0.010685156099498272, 0.03624529018998146]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.035339221358299255, 0.010706206783652306, -0.001701260800473392, -0.00862252525985241, 0.006445988081395626, 0.08198338001966476, -0.022678885608911514, 0.01434261817485094, -0.008092232048511505, 0.03345781937241554]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.030748076736927032, 0.009340512566268444, -0.013637945055961609, 0.011183148249983788, -0.013879665173590183, 0.046350326389074326, -0.024090109393000603, 0.02885228954255581, -0.01699884608387947, 0.01672385260462761]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.040792081505060196, -0.00872269831597805, -0.015838179737329483, -0.03141209855675697, -7.104632823029533e-05, 0.08301416039466858, -0.034691162407398224, 0.0026397297624498606, 0.009255227632820606, 0.05415954813361168]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.02156883291900158, 0.003969342447817326, -0.030446071177721024, 0.008231461979448795, -0.01271845493465662, 0.03793857619166374, -0.013524272479116917, -0.0385628417134285, -0.0058258213102817535, 0.03505263477563858]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.027544165030121803, -0.01773364469408989, -0.013286487199366093, -0.008328652940690517, -0.011047529056668282, 0.05237515643239021, -0.016948163509368896, 0.02806701697409153, -0.0018120920285582542, 0.027241172268986702]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [-0.03464886546134949, -0.003521248232573271, -0.010239562019705772, -0.018618224188685417, 0.004094886127859354, 0.062059685587882996, -0.013881963677704334, -0.0008639032603241503, -0.029874088242650032, 0.033531222492456436]}\n",
+            "Embedding shape: 10\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "You can pass additional arguments that are supported by `sentence-transformer` models, such as `convert_to_numpy=False`. These arguments are passed as a `dict` to the `SentenceTransformerEmbeddings` transform by using the `inference_args` parameter.\n",
+        "\n",
+        "When you pass `convert_to_numpy=False`, the output contains `torch.Tensor` matrices."
+      ],
+      "metadata": {
+        "id": "1MFom0PW_vRv"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artifact_location_t5_with_inference_args = tempfile.mkdtemp(prefix='huggingface_')\n",
+        "\n",
+        "embedding_transform = SentenceTransformerEmbeddings(\n",
+        "        model_name=text_embedding_model_name, columns=['x'],\n",
+        "        inference_args={'convert_to_numpy': False}\n",
+        "        )\n",
+        "\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = (\n",
+        "      pipeline\n",
+        "      | \"CreateData\" >> beam.Create(content))\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location_t5_with_inference_args).with_transform(embedding_transform))\n",
+        "\n",
+        "  # The outputs are in the PyTorch tensor type.\n",
+        "  transformed_pcoll | 'LogOutput' >> beam.Map(lambda x: print(type(x['x'])))\n",
+        "\n",
+        "  transformed_pcoll | \"PrintEmbeddingShape\" >> beam.Map(lambda x: print(f\"Embedding shape: {len(x['x'])}\"))\n"
+      ],
+      "metadata": {
+        "id": "xyezKuzY_uLD",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "96babb3b-c61b-40a2-96f7-572a2a46bd83"
+      },
+      "execution_count": 24,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n",
+            "<class 'torch.Tensor'>\n",
+            "Embedding shape: 768\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Use MLTransform in read mode\n",
+        "\n",
+        "In `read` mode, `MLTransform` uses the artifacts generated during `write` mode. In this case, the `SentenceTransformEmbedding` transform and its attributes are loaded from the saved artifacts. You don't need to specify the artifacts again during `read` mode.\n",
+        "\n",
+        "In this way, `MLTransform` provides consistent preprocessing steps for training and inference workloads."
+      ],
+      "metadata": {
+        "id": "aPIQzCoF_EBj"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "test_content = [\n",
+        "    {\n",
+        "        'x': 'This is a test sentence'\n",
+        "    },\n",
+        "    {\n",
+        "        'x': 'The park is full of dogs'\n",
+        "    },\n",
+        "    {\n",
+        "        'x': \"Should I sign up for Medicare Part B if I have Veterans' Benefits?\"\n",
+        "    }\n",
+        "]\n",
+        "\n",
+        "# Uses the T5 model to generate text embeddings\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = (\n",
+        "      pipeline\n",
+        "      | \"CreateData\" >> beam.Create(test_content))\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(read_artifact_location=artifact_location_t5))\n",
+        "\n",
+        "  transformed_pcoll | beam.Map(truncate_embeddings) | 'LogOutput' >> beam.Map(print)\n"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "RCqYeUd3_F3C",
+        "outputId": "4c5b61d2-df39-4af3-8520-f529f243e3b1"
+      },
+      "execution_count": 26,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'x': [0.00036313451710157096, -0.03929319977760315, -0.03574873134493828, 0.05015222355723381, 0.04295048117637634, 0.04800170287489891, 0.006883862894028425, -0.02567591704428196, -0.048067063093185425, 0.036534328013658524]}\n",
+            "{'x': [-0.053793832659721375, 0.006730600260198116, -0.025130020454525948, 0.04363932088017464, 0.03323192894458771, 0.008803879842162132, -0.015412433072924614, 0.008926985785365105, -0.061175212264060974, 0.04573329910635948]}\n",
+            "{'x': [-0.03464885801076889, -0.003521254053339362, -0.010239563882350922, -0.018618224188685417, 0.004094892647117376, 0.062059689313173294, -0.013881963677704334, -0.000863900815602392, -0.029874078929424286, 0.03353121876716614]}\n"
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/examples/notebooks/beam-ml/data_preprocessing/scale_data.ipynb b/examples/notebooks/beam-ml/data_preprocessing/scale_data.ipynb
new file mode 100644
index 000000000000..3c8946362a31
--- /dev/null
+++ b/examples/notebooks/beam-ml/data_preprocessing/scale_data.ipynb
@@ -0,0 +1,504 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "A7HK1Duzdnew"
+      },
+      "outputs": [],
+      "source": [
+        "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n",
+        "\n",
+        "# Licensed to the Apache Software Foundation (ASF) under one\n",
+        "# or more contributor license agreements. See the NOTICE file\n",
+        "# distributed with this work for additional information\n",
+        "# regarding copyright ownership. The ASF licenses this file\n",
+        "# to you under the Apache License, Version 2.0 (the\n",
+        "# \"License\"); you may not use this file except in compliance\n",
+        "# with the License. You may obtain a copy of the License at\n",
+        "#\n",
+        "#   http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing,\n",
+        "# software distributed under the License is distributed on an\n",
+        "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n",
+        "# KIND, either express or implied. See the License for the\n",
+        "# specific language governing permissions and limitations\n",
+        "# under the License"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Use MLTransform to scale data\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/scale_data.ipynb.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/colab_32px.png\" />Run in Google Colab</a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/scale_data.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/github_32px.png\" />View source on GitHub</a>\n",
+        "  </td>\n",
+        "</table>\n"
+      ],
+      "metadata": {
+        "id": "ZUSiAR62SgO8"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Scaling data is an important preprocessing step for training machine learning (ML) models, because it helps to ensure that all features have a similar weight or influence on the model. The following are benefits of scaling data:\n",
+        "\n",
+        "* **Improved convergence of gradient descent algorithms:** Many machine learning algorithms, such as linear regression and neural networks, use gradient descent to optimize their parameters. Gradient descent iteratively moves the parameters of the model in the direction that reduces the loss function. If the features aren't scaled, features with larger ranges can have a much larger impact on the gradient, making it difficult for the model to converge. Scaling the features helps to ensure that all features contribute equally to the gradient, which can lead to faster and more stable convergence.\n",
+        "\n",
+        "* **Uniformity in features:** If one feature has a much larger range than the other features, it can dominate the model and make it difficult for the model to learn from the other features. This lack of uniformity can cause poor performance and biased predictions. Scaling the features brings all of the features into a similar range.\n",
+        "\n",
+        "\n",
+        "To scale your dataset using Apache Beam, use `MLTransform` with one of the following transforms:\n",
+        "\n",
+        "* `ScaleTo01`: Calculates the minimum and maximum of an entire dataset, and then scales the dataset between 0 and 1 based on minimum and maximum values.\n",
+        "* `ScaleToZScore`: Calculates the mean and variance of an entire dataset, and then scales the dataset based on those values.\n",
+        "* `ScaleByMinMax`: Scales the data in a dataset, taking minimum and maximum values as input parameters.\n",
+        "\n",
+        "For each data processing transform, `MLTransform` runs in both `write` mode and `read` mode. For more information about using `MLTransform`, see [Preprocess data with MLTransform](https://beam.apache.org/documentation/ml/preprocess-data/) in the Apache Beam documentation.\n",
+        "\n",
+        "## MLTransform in write mode\n",
+        "\n",
+        "When `MLTransform` is in `write` mode, it produces artifacts, such as minimum, maximum, and variance, for different data processing transforms. These artifacts allow you to ensure that you're applying the same artifacts, and any other preprocessing transforms, when you train your model and serve it in production, or when you test its accuracy.\n",
+        "\n",
+        "## MLTransform in read mode\n",
+        "\n",
+        "In read mode, `MLTransform` uses the artifacts generated in `write` mode to scale the entire dataset."
+      ],
+      "metadata": {
+        "id": "F8-yTcjPUHLv"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Import the required modules\n",
+        "\n",
+        "To use `MLTransfrom`, install `tensorflow_transform` and the Apache Beam SDK version 2.53.0 or later."
+      ],
+      "metadata": {
+        "id": "8qhhZTbIyqFb"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "! pip install apache_beam>=2.53.0 --quiet\n",
+        "! pip install tensorflow-transform --quiet"
+      ],
+      "metadata": {
+        "id": "cFP4CQksT_Zv"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "import tempfile\n",
+        "import apache_beam as beam\n",
+        "from apache_beam.ml.transforms.base import MLTransform\n",
+        "from apache_beam.ml.transforms.tft import ScaleTo01\n",
+        "from apache_beam.ml.transforms.tft import ScaleByMinMax\n",
+        "from apache_beam.ml.transforms.tft import ScaleToZScore"
+      ],
+      "metadata": {
+        "id": "y-B7OZ0lyeee"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artifact_location_scale_to_01 = tempfile.mkdtemp(prefix='scale_to_01_')\n",
+        "artifact_location_scale_to_zscore = tempfile.mkdtemp(prefix='scale_to_zscore_')\n",
+        "artifact_location_scale_by_min_max = tempfile.mkdtemp(prefix='scale_by_min_max_')"
+      ],
+      "metadata": {
+        "id": "UyBji8pSUEJt"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# data used in MLTransform's write mode\n",
+        "data = [\n",
+        "    {'int_feature_1' : 11, 'int_feature_2': -10},\n",
+        "    {'int_feature_1': 34, 'int_feature_2': -33},\n",
+        "    {'int_feature_1': 5, 'int_feature_2': -63},\n",
+        "    {'int_feature_1': 12, 'int_feature_2': -38},\n",
+        "    {'int_feature_1': 32, 'int_feature_2': -65},\n",
+        "    {'int_feature_1': 63, 'int_feature_2': -21},\n",
+        "]\n",
+        "\n",
+        "# data used in MLTransform's read mode\n",
+        "test_data = [\n",
+        "    {'int_feature_1': 29, 'int_feature_2': -20},\n",
+        "    {'int_feature_1': -5, 'int_feature_2': -11},\n",
+        "    {'int_feature_1': 5, 'int_feature_2': -44},\n",
+        "    {'int_feature_1': 29, 'int_feature_2': -12},\n",
+        "    {'int_feature_1': 20, 'int_feature_2': -53},\n",
+        "    {'int_feature_1': 70, 'int_feature_2': -8}\n",
+        "]\n"
+      ],
+      "metadata": {
+        "id": "lNG0WI7Dy1dd"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Scale the data between 0 and 1\n",
+        "\n",
+        "Scale the data so that it's in the range of 0 to 1. To scale the data, the transform calculates minimum and maximum values on the whole dataset, and then performs the following calculation:\n",
+        "\n",
+        "`x = (x - x_min) / (x_max - x_min)`\n",
+        "\n",
+        "To scale the data, use the [ScaleTo01](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ScaleTo01) data processing transform in `MLTransform`."
+      ],
+      "metadata": {
+        "id": "MjMPgB1T0Wzu"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# MLTransform in write mode.\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(data)\n",
+        "\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location_scale_to_01).with_transform(\n",
+        "          ScaleTo01(columns=['int_feature_1', 'int_feature_2'])\n",
+        "      )\n",
+        "  )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 211
+        },
+        "id": "xuoLla2u0Cf_",
+        "outputId": "dab7f024-ca5e-4f2b-83b8-c09612c19450"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "\n",
+              "        if (typeof window.interactive_beam_jquery == 'undefined') {\n",
+              "          var jqueryScript = document.createElement('script');\n",
+              "          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n",
+              "          jqueryScript.type = 'text/javascript';\n",
+              "          jqueryScript.onload = function() {\n",
+              "            var datatableScript = document.createElement('script');\n",
+              "            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n",
+              "            datatableScript.type = 'text/javascript';\n",
+              "            datatableScript.onload = function() {\n",
+              "              window.interactive_beam_jquery = jQuery.noConflict(true);\n",
+              "              window.interactive_beam_jquery(document).ready(function($){\n",
+              "                \n",
+              "              });\n",
+              "            }\n",
+              "            document.head.appendChild(datatableScript);\n",
+              "          };\n",
+              "          document.head.appendChild(jqueryScript);\n",
+              "        } else {\n",
+              "          window.interactive_beam_jquery(document).ready(function($){\n",
+              "            \n",
+              "          });\n",
+              "        }"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(int_feature_1=array([0.10344828], dtype=float32), int_feature_2=array([1.], dtype=float32))\n",
+            "Row(int_feature_1=array([0.5], dtype=float32), int_feature_2=array([0.58181816], dtype=float32))\n",
+            "Row(int_feature_1=array([0.], dtype=float32), int_feature_2=array([0.03636364], dtype=float32))\n",
+            "Row(int_feature_1=array([0.12068965], dtype=float32), int_feature_2=array([0.4909091], dtype=float32))\n",
+            "Row(int_feature_1=array([0.46551725], dtype=float32), int_feature_2=array([0.], dtype=float32))\n",
+            "Row(int_feature_1=array([1.], dtype=float32), int_feature_2=array([0.8], dtype=float32))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "In the this dataset, the following are the minimum and maximum values for the columns:\n",
+        "* `int_feature_1`: 5 and 63.\n",
+        "* `int_feature_2`: -65 and -10\n",
+        "\n",
+        "In the output for the column `int_feature_1`, the data is scaled between 0 and 1 by using the values `5` and `63`. `5` is scaled to `0`, and `63` is scaled to `1`. The remaining values are scaled between 0 and 1 by using the formula `x = (x - x_min) / (x_max - x_min)`."
+      ],
+      "metadata": {
+        "id": "Hwvh4UyI23pz"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# MLTransform in read mode\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(test_data)\n",
+        "\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(read_artifact_location=artifact_location_scale_to_01)\n",
+        "  )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "id": "M8obCCCL1TM6",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "813bf5c2-6be7-49d3-d622-33f3f66fe2c9"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(int_feature_1=array([0.41379312], dtype=float32), int_feature_2=array([0.8181818], dtype=float32))\n",
+            "Row(int_feature_1=array([-0.1724138], dtype=float32), int_feature_2=array([0.9818182], dtype=float32))\n",
+            "Row(int_feature_1=array([0.], dtype=float32), int_feature_2=array([0.38181818], dtype=float32))\n",
+            "Row(int_feature_1=array([0.41379312], dtype=float32), int_feature_2=array([0.96363634], dtype=float32))\n",
+            "Row(int_feature_1=array([0.25862068], dtype=float32), int_feature_2=array([0.21818182], dtype=float32))\n",
+            "Row(int_feature_1=array([1.1206896], dtype=float32), int_feature_2=array([1.0363636], dtype=float32))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "`MLTransform` learned in `write` mode that `int_feature_1` ranges from 5 to 63.\n",
+        "\n",
+        "In `read` mode, when it encounters 29 in `test_data` for `int_feature_1`, it scales it by using the following formula:\n",
+        "```\n",
+        "(value - min) / (max - min)\n",
+        "```\n",
+        "\n",
+        "The following calculation shows the formula with the values:\n",
+        "```\n",
+        "(29 - 5) / (63 - 5) = 0.41379312\n",
+        "```\n",
+        "\n",
+        "Twenty-nine is scaled based on the minimum and maximum values generated in `write` mode.\n"
+      ],
+      "metadata": {
+        "id": "jwSiMoAH4vq5"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Scale by using the z-score\n",
+        "\n",
+        "Similar to `ScaleTo01`, use [ScaleToZScore](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html#apache_beam.ml.transforms.tft.ScaleToZScore) to scale the values by using the [z-score](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/scale_to_z_score#:~:text=Scaling%20to%20z%2Dscore%20subtracts%20out%20the%20mean%20and%20divides%20by%20standard%20deviation.%20Note%20that%20the%20standard%20deviation%20computed%20here%20is%20based%20on%20the%20biased%20variance%20(0%20delta%20degrees%20of%20freedom)%2C%20as%20computed%20by%20analyzers.var).\n"
+      ],
+      "metadata": {
+        "id": "VH1CmCGm_PtS"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# MLTransform in write mode\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(data)\n",
+        "\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location_scale_to_zscore).with_transform(\n",
+        "          ScaleToZScore(columns=['int_feature_1', 'int_feature_2'])\n",
+        "      )\n",
+        "  )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "OOrnLBbD_Qe4",
+        "outputId": "4754b0ac-dc40-4a55-c999-33270e785acc"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(int_feature_1=array([-0.76950264], dtype=float32), int_feature_2=array([1.401755], dtype=float32))\n",
+            "Row(int_feature_1=array([0.3974355], dtype=float32), int_feature_2=array([0.2638597], dtype=float32))\n",
+            "Row(int_feature_1=array([-1.0739213], dtype=float32), int_feature_2=array([-1.2203515], dtype=float32))\n",
+            "Row(int_feature_1=array([-0.7187662], dtype=float32), int_feature_2=array([0.01649117], dtype=float32))\n",
+            "Row(int_feature_1=array([0.2959626], dtype=float32), int_feature_2=array([-1.3192989], dtype=float32))\n",
+            "Row(int_feature_1=array([1.8687923], dtype=float32), int_feature_2=array([0.8575442], dtype=float32))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# MLTransform in read mode\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(test_data)\n",
+        "\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(read_artifact_location=artifact_location_scale_to_zscore)\n",
+        "  )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "lLXGBaOtCRv-",
+        "outputId": "3ef4d142-0be2-4350-b2c4-3b676be74cee"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(int_feature_1=array([0.14375328], dtype=float32), int_feature_2=array([0.9070179], dtype=float32))\n",
+            "Row(int_feature_1=array([-1.5812857], dtype=float32), int_feature_2=array([1.3522812], dtype=float32))\n",
+            "Row(int_feature_1=array([-1.0739213], dtype=float32), int_feature_2=array([-0.28035107], dtype=float32))\n",
+            "Row(int_feature_1=array([0.14375328], dtype=float32), int_feature_2=array([1.3028076], dtype=float32))\n",
+            "Row(int_feature_1=array([-0.31287467], dtype=float32), int_feature_2=array([-0.7256144], dtype=float32))\n",
+            "Row(int_feature_1=array([2.2239475], dtype=float32), int_feature_2=array([1.5007024], dtype=float32))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Scale by using ScaleByMinMax\n",
+        "\n",
+        "Use  [ScaleByMinMax](https://github.com/apache/beam/blob/9e8a310f0c0faddfba28176df5893d8ad8fd10a0/sdks/python/apache_beam/ml/transforms/tft.py#L450) to scale your data into the range of `[min_value, max_value]`."
+      ],
+      "metadata": {
+        "id": "McISy8QkC00v"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "min_value = 1\n",
+        "max_value = 10\n",
+        "\n",
+        "# MLTransform in write mode\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(data)\n",
+        "\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location_scale_by_min_max).with_transform(\n",
+        "          ScaleByMinMax(columns=['int_feature_1', 'int_feature_2'], min_value=min_value, max_value=max_value)\n",
+        "      )\n",
+        "  )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "cpn_wKQ3Cxo0",
+        "outputId": "20be1e2b-36a2-4bae-dc53-5e5b5f1692be"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(int_feature_1=array([1.9310346], dtype=float32), int_feature_2=array([10.], dtype=float32))\n",
+            "Row(int_feature_1=array([5.5], dtype=float32), int_feature_2=array([6.2363634], dtype=float32))\n",
+            "Row(int_feature_1=array([1.], dtype=float32), int_feature_2=array([1.3272727], dtype=float32))\n",
+            "Row(int_feature_1=array([2.086207], dtype=float32), int_feature_2=array([5.418182], dtype=float32))\n",
+            "Row(int_feature_1=array([5.1896553], dtype=float32), int_feature_2=array([1.], dtype=float32))\n",
+            "Row(int_feature_1=array([10.], dtype=float32), int_feature_2=array([8.200001], dtype=float32))\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# MLTransform in read mode\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = pipeline | \"CreateData\" >> beam.Create(test_data)\n",
+        "\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(read_artifact_location=artifact_location_scale_by_min_max)\n",
+        "  )\n",
+        "  transformed_pcoll | \"Print\" >> beam.Map(print)"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "3aQjFPzBI7dS",
+        "outputId": "e9967889-27eb-4295-dfad-a285d670826c"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Row(int_feature_1=array([4.7241383], dtype=float32), int_feature_2=array([8.363636], dtype=float32))\n",
+            "Row(int_feature_1=array([-0.5517242], dtype=float32), int_feature_2=array([9.836364], dtype=float32))\n",
+            "Row(int_feature_1=array([1.], dtype=float32), int_feature_2=array([4.4363637], dtype=float32))\n",
+            "Row(int_feature_1=array([4.7241383], dtype=float32), int_feature_2=array([9.672727], dtype=float32))\n",
+            "Row(int_feature_1=array([3.3275862], dtype=float32), int_feature_2=array([2.9636364], dtype=float32))\n",
+            "Row(int_feature_1=array([11.086206], dtype=float32), int_feature_2=array([10.327272], dtype=float32))\n"
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb b/examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb
new file mode 100644
index 000000000000..ed4da57e297b
--- /dev/null
+++ b/examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb
@@ -0,0 +1,308 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "muiqKarukWj0"
+      },
+      "outputs": [],
+      "source": [
+        "# @title ###### Licensed to the Apache Software Foundation (ASF), Version 2.0 (the \"License\")\n",
+        "\n",
+        "# Licensed to the Apache Software Foundation (ASF) under one\n",
+        "# or more contributor license agreements. See the NOTICE file\n",
+        "# distributed with this work for additional information\n",
+        "# regarding copyright ownership. The ASF licenses this file\n",
+        "# to you under the Apache License, Version 2.0 (the\n",
+        "# \"License\"); you may not use this file except in compliance\n",
+        "# with the License. You may obtain a copy of the License at\n",
+        "#\n",
+        "#   http://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing,\n",
+        "# software distributed under the License is distributed on an\n",
+        "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n",
+        "# KIND, either express or implied. See the License for the\n",
+        "# specific language governing permissions and limitations\n",
+        "# under the License"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# Generate text embeddings by using the Vertex AI API\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://colab.sandbox.google.com/github/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/colab_32px.png\" />Run in Google Colab</a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a target=\"_blank\" href=\"https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/data_preprocessing/vertex_ai_text_embeddings.ipynb\"><img src=\"https://raw.githubusercontent.com/google/or-tools/main/tools/github_32px.png\" />View source on GitHub</a>\n",
+        "  </td>\n",
+        "</table>\n"
+      ],
+      "metadata": {
+        "id": "ZUSiAR62SgO8"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "Text embeddings are a way to represent text as numerical vectors. This process lets computers understand and process text data, which is essential for many natural language processing (NLP) tasks.\n",
+        "\n",
+        "The following NLP tasks use embeddings:\n",
+        "\n",
+        "* **Semantic search:** Find documents or passages that are relevant to a query when the query doesn't use the exact same words as the documents.\n",
+        "* **Text classification:** Categorize text data into different classes, such as spam and not spam, or positive sentiment and negative sentiment.\n",
+        "* **Machine translation:** Translate text from one language to another and preserve the meaning.\n",
+        "* **Text summarization:** Create shorter summaries of text.\n",
+        "\n",
+        "This notebook uses the Vertex AI text-embeddings API to generate text embeddings that use Google’s large generative artificial intelligence (AI) models. To generate text embeddings by using the Vertex AI text-embeddings API, use `MLTransform` with the `VertexAITextEmbeddings` class to specify the model configuration. For more information, see [Get text embeddings](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings) in the Vertex AI documentation. \n",
+        "\n",
+        "For more information about using `MLTransform`, see [Preprocess data with MLTransform](https://beam.apache.org/documentation/ml/preprocess-data/) in the Apache Beam documentation.\n",
+        "\n",
+        "## Requirements\n",
+        "\n",
+        "To use the Vertex AI text-embeddings API, complete the following prerequisites:\n",
+        "\n",
+        "* Install the `google-cloud-aiplatform` Python package.\n",
+        "* Do one of the following tasks:\n",
+        "  * Configure credentials for your Google Cloud project. For more information, see [Google Auth Library for Python](https://googleapis.dev/python/google-auth/latest/reference/google.auth.html#module-google.auth).\n",
+        "  * Store the path to a service account JSON file by using the [GOOGLE_APPLICATION_CREDENTIALS](https://cloud.google.com/docs/authentication/application-default-credentials#GAC) environment variable."
+      ],
+      "metadata": {
+        "id": "bkpSCGCWlqAf"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "To use your Google Cloud account, authenticate this notebook."
+      ],
+      "metadata": {
+        "id": "W29FgO5Qv2ew"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "from google.colab import auth\n",
+        "auth.authenticate_user()\n",
+        "\n",
+        "project = '<PROJECT_ID>' # Replace <PROJECT_ID> with a valid Google Cloud project ID."
+      ],
+      "metadata": {
+        "id": "nYyyGYt3licq"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Install dependencies\n",
+        " Install Apache Beam and the dependencies required for the Vertex AI text-embeddings API."
+      ],
+      "metadata": {
+        "id": "UQROd16ZDN5y"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "! pip install apache_beam[gcp]>=2.53.0 --quiet"
+      ],
+      "metadata": {
+        "id": "BTxob7d5DLBM"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import tempfile\n",
+        "import apache_beam as beam\n",
+        "from apache_beam.ml.transforms.base import MLTransform\n",
+        "from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAITextEmbeddings"
+      ],
+      "metadata": {
+        "id": "SkMhR7H6n1P0"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Transform the data\n",
+        "\n",
+        "`MLTransform` is a `PTransform` that you can use for data preparation, including generating text embeddings.\n",
+        "\n",
+        "### Use MLTransform in write mode\n",
+        "\n",
+         "In `write` mode, `MLTransform` saves the transforms and their attributes to an artifact location. Then, when you run `MLTransform` in `read` mode, these transforms are used. This process ensures that you're applying the same preprocessing steps when you train your model and when you serve the model in production or test its accuracy."
+      ],
+      "metadata": {
+        "id": "cokOaX2kzyke"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Get the data\n",
+        "\n",
+        "`MLTransform` processes dictionaries that include column names and their associated text data. To generate embeddings for specific columns, specify these column names in the `columns` argument of `VertexAITextEmbeddings`. This transform uses the the Vertex AI text-embeddings API for online predictions to generate an embeddings vector for each sentence."
+      ],
+      "metadata": {
+        "id": "-x7fVvuy-aDs"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "artifact_location = tempfile.mkdtemp(prefix='vertex_ai')\n",
+        "\n",
+        "# Use the latest text embedding model from the Vertex AI text-embeddings API documentation.\n",
+        "# https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/text-embeddings\n",
+        "text_embedding_model_name = 'textembedding-gecko@latest'\n",
+        "\n",
+        "# Generate text embeddings on the sentences.\n",
+        "content = [\n",
+        "    {\n",
+        "        'x' : 'I would like embeddings for this text'\n",
+        "    },\n",
+        "    {\n",
+        "        'x' : 'Hello world'\n",
+        "    },\n",
+        "    {\n",
+        "        'x': 'The Dog is running in the park.'\n",
+        "    }\n",
+        "  ]\n",
+        "\n",
+        "# helper function that returns a dict containing only first\n",
+        "# ten elements of generated embeddings\n",
+        "def truncate_embeddings(d):\n",
+        "  for key in d.keys():\n",
+        "    d[key] = d[key][:10]\n",
+        "  return d"
+      ],
+      "metadata": {
+        "id": "be-vR159pylF"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "embedding_transform = VertexAITextEmbeddings(\n",
+        "        model_name=text_embedding_model_name, columns=['x'], project=project)\n",
+        "\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = (\n",
+        "      pipeline\n",
+        "      | \"CreateData\" >> beam.Create(content))\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(write_artifact_location=artifact_location).with_transform(embedding_transform))\n",
+        "\n",
+        "  # Show only the first ten elements of the embeddings to prevent clutter in the output.\n",
+        "  transformed_pcoll | beam.Map(truncate_embeddings) | 'LogOutput' >> beam.Map(print)\n",
+        "\n",
+        "  transformed_pcoll | \"PrintEmbeddingShape\" >> beam.Map(lambda x: print(f\"Embedding shape: {len(x['x'])}\"))"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "UQGm1be3p7lM",
+        "outputId": "b41172ca-1c73-4952-ca87-bfe45ca88a6c"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'x': [0.041293490678071976, -0.010302993468940258, -0.048611514270305634, -0.01360565796494484, 0.06441926211118698, 0.022573700174689293, 0.016446372494101524, -0.033894773572683334, 0.004581860266625881, 0.060710687190294266]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [0.05889148637652397, -0.0046180677600204945, -0.06738516688346863, -0.012708292342722416, 0.06461101770401001, 0.025648491457104683, 0.023468563333153725, -0.039828114211559296, -0.009968819096684456, 0.050098177045583725]}\n",
+            "Embedding shape: 10\n",
+            "{'x': [0.04683901369571686, -0.013076924718916416, -0.082594133913517, -0.01227626483887434, 0.00417641457170248, -0.024504298344254494, 0.04282262548804283, -0.0009824123699218035, -0.02860993705689907, 0.01609829254448414]}\n",
+            "Embedding shape: 10\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### Use MLTransform in read mode\n",
+        "\n",
+        "In `read` mode, `MLTransform` uses the artifacts saved during `write` mode. In this example, the transform and its attributes are loaded from the saved artifacts. You don't need to specify artifacts again during `read` mode.\n",
+        "\n",
+        "In this way, `MLTransform` provides consistent preprocessing steps for training and inference workloads."
+      ],
+      "metadata": {
+        "id": "JLkmQkiLx_6h"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "test_content = [\n",
+        "    {\n",
+        "        'x': 'This is a test sentence'\n",
+        "    },\n",
+        "    {\n",
+        "        'x': 'The park is full of dogs'\n",
+        "    },\n",
+        "]\n",
+        "\n",
+        "with beam.Pipeline() as pipeline:\n",
+        "  data_pcoll = (\n",
+        "      pipeline\n",
+        "      | \"CreateData\" >> beam.Create(test_content))\n",
+        "  transformed_pcoll = (\n",
+        "      data_pcoll\n",
+        "      | \"MLTransform\" >> MLTransform(read_artifact_location=artifact_location))\n",
+        "\n",
+        "  transformed_pcoll | beam.Map(truncate_embeddings) | 'LogOutput' >> beam.Map(print)\n"
+      ],
+      "metadata": {
+        "id": "r8Y5vgfLx_Xu",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "e7cbf6b7-5c31-4efa-90cf-7a8a108ecc77"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "{'x': [0.04782044142484665, -0.010078949853777885, -0.05793016776442528, -0.026060665026307106, 0.05756739526987076, 0.02292264811694622, 0.014818413183093071, -0.03718176111578941, -0.005486017093062401, 0.04709304869174957]}\n",
+            "{'x': [0.042911216616630554, -0.007554919924587011, -0.08996245265007019, -0.02607591263949871, 0.0008614308317191899, -0.023671219125390053, 0.03999944031238556, -0.02983051724731922, -0.015057179145514965, 0.022963201627135277]}\n"
+          ]
+        }
+      ]
+    }
+  ]
+}
diff --git a/examples/notebooks/beam-ml/mltransform_basic.ipynb b/examples/notebooks/beam-ml/mltransform_basic.ipynb
index 43068624eb82..b0af96d08593 100644
--- a/examples/notebooks/beam-ml/mltransform_basic.ipynb
+++ b/examples/notebooks/beam-ml/mltransform_basic.ipynb
@@ -55,7 +55,7 @@
         "id": "d3b81cf2-8603-42bd-995e-9e14631effd0"
       },
       "source": [
-        "This notebook demonstrates how to use `MLTransform` to preprocess data for machine learning workflows. Apache Beam provides a set of transforms for preprocessing data for training and inference. The `MLTransform` class wraps various transforms in one PTransform, simplifying your workflow. For a list of available preprocessing transforms see the [Preprocess data with MLTransform](https://beam.apache.org/documentation/ml/preprocess-data/#transforms) page in Apache Beam documentation.\n",
+        "This notebook demonstrates how to use `MLTransform` to preprocess data for machine learning workflows. Apache Beam provides a set of transforms for preprocessing data for training and inference. The `MLTransform` class wraps various transforms in one `PTransform`, simplifying your workflow. For a list of available preprocessing transforms see the [Preprocess data with MLTransform](https://beam.apache.org/documentation/ml/preprocess-data/#transforms) page in the Apache Beam documentation.\n",
         "\n",
         "This notebook uses data processing transforms defined in the [apache_beam/ml/transforms/tft](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.transforms.tft.html) module."
       ]
diff --git a/examples/notebooks/beam-ml/per_key_models.ipynb b/examples/notebooks/beam-ml/per_key_models.ipynb
index 53845c0b3e19..3e71c1d119a2 100644
--- a/examples/notebooks/beam-ml/per_key_models.ipynb
+++ b/examples/notebooks/beam-ml/per_key_models.ipynb
@@ -402,7 +402,7 @@
     {
       "cell_type": "markdown",
       "source": [
-        "Use the formatted keys to define a `KeyedModelHandler` that maps keys to the `ModelHandler` used for those keys. The `KeyedModelHandler` method lets you define an optional `max_models_per_worker_hint`, which limits the number of models that can be held in a single worker process at one time. If your worker might run out of memory, use this option. For more information about managing memory, see [Use a keyed ModelHandler](https://beam.apache.org/documentation/sdks/python-machine-learning/index.html#use-a-keyed-modelhandler)."
+        "Use the formatted keys to define a `KeyedModelHandler` that maps keys to the `ModelHandler` used for those keys. The `KeyedModelHandler` method lets you define an optional `max_models_per_worker_hint`, which limits the number of models that can be held in a single worker process at one time. If your worker might run out of memory, use this option. For more information about managing memory, see [Use a keyed ModelHandler](https://beam.apache.org/documentation/ml/about-ml/#use-a-keyed-modelhandler-object)."
       ],
       "metadata": {
         "id": "IP65_5nNGIb8"
diff --git a/examples/notebooks/beam-ml/run_custom_inference.ipynb b/examples/notebooks/beam-ml/run_custom_inference.ipynb
index a66c5847de0e..dde42399ff35 100644
--- a/examples/notebooks/beam-ml/run_custom_inference.ipynb
+++ b/examples/notebooks/beam-ml/run_custom_inference.ipynb
@@ -60,7 +60,7 @@
         "NLP locates named entities in unstructured text and classifies the entities using pre-defined labels, such as person name, organization, date, and so on.\n",
         "\n",
         "This example illustrates how to use the popular `spaCy` package to load a machine learning (ML) model and perform inference in an Apache Beam pipeline using the RunInference `PTransform`.\n",
-        "For more information about the RunInference API, see [Machine Learning](https://beam.apache.org/documentation/sdks/python-machine-learning) in the Apache Beam documentation."
+        "For more information about the RunInference API, see [About Beam ML](https://beam.apache.org/documentation/ml/about-ml) in the Apache Beam documentation."
       ]
     },
     {
@@ -104,7 +104,7 @@
         "\n",
         "To learn more about `spaCy`, create a `spaCy` language object in memory using `spaCy`'s trained models.\n",
         "You can install these models as Python packages.\n",
-        "For more inforamtion, see spaCy's [Models and Languages](https://spacy.io/usage/models) documentation."
+        "For more information, see spaCy's [Models and Languages](https://spacy.io/usage/models) documentation."
       ]
     },
     {
diff --git a/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb b/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb
index a314f6cd7116..115b70b11e94 100644
--- a/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb
+++ b/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb
@@ -71,7 +71,7 @@
         "You can use Apache Beam versions 2.40.0 and later with the [RunInference API](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.RunInference) for local and remote inference with batch and streaming pipelines.\n",
         "The RunInference API leverages Apache Beam concepts, such as the `BatchElements` transform and the `Shared` class, to support models in your pipelines that create transforms optimized for machine learning inference.\n",
         "\n",
-        "For more information about the RunInference API, see [Machine Learning](https://beam.apache.org/documentation/sdks/python-machine-learning) in the Apache Beam documentation."
+        "For more information about the RunInference API, see [About Beam ML](https://beam.apache.org/documentation/ml/about-ml) in the Apache Beam documentation."
       ]
     },
     {
@@ -221,7 +221,7 @@
         "    # Pad the token tensors to max length to make sure that all of the tensors\n",
         "    # are of the same length and stack-able by the RunInference API. Normally, you would batch first\n",
         "    # then tokenize the batch, padding each tensor the max length in the batch.\n",
-        "    # See: https://beam.apache.org/documentation/sdks/python-machine-learning/#unable-to-batch-tensor-elements\n",
+        "    # See: https://beam.apache.org/documentation/ml/about-ml/#unable-to-batch-tensor-elements\n",
         "    tokens = self._tokenizer(text_input, return_tensors='pt', padding='max_length', max_length=512)\n",
         "    # Squeeze because tokenization adds an extra dimension, which is empty,\n",
         "    # in this case because we tokenize one element at a time.\n",
diff --git a/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb b/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb
index 098cb150bfd2..c2dfb06a6e67 100644
--- a/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb
+++ b/examples/notebooks/beam-ml/speech_emotion_tensorflow.ipynb
@@ -132,7 +132,7 @@
         "* **[IPython](https://ipython.readthedocs.io/en/stable/index.html)**: Creates visualizations for multimedia content. Here we have used it for playing audio files.\n",
         "* **[Sklearn](https://scikit-learn.org/stable/index.html)**: Offers comprehensive tools for Machine Learning. Here we have used it for preprocessing and splitting the data.\n",
         "* **[TensorFlow](https://www.tensorflow.org/api_docs)** and **[Keras](https://keras.io/api/)**: Enables building and training complex Machine Learning and Deep Learning model.\n",
-        "* **[TFModelHandlerNumpy](https://beam.apache.org/documentation/sdks/python-machine-learning/#tensorflow)**: Defines the configuration used to load/use the model that we train. We use TFModelHandlerNumpy because the model was trained with TensorFlow and takes numpy arrays as input.\n",
+        "* **[TFModelHandlerNumpy](https://beam.apache.org/documentation/ml/about-ml/#tensorflow)**: Defines the configuration used to load/use the model that we train. We use TFModelHandlerNumpy because the model was trained with TensorFlow and takes numpy arrays as input.\n",
         "* **[RunInference](https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.html#apache_beam.ml.inference.RunInference)**: Loads the model and obtains predictions as part of the Apache Beam pipeline. For more information, see docs on prediction and inference.\n",
         "* **[Apache Beam](https://beam.apache.org/documentation/)**: Builds a pipeline for Image Processing."
       ]
diff --git a/gradle.properties b/gradle.properties
index ef84ea9a5cc9..8f8736a100f6 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -30,8 +30,8 @@ signing.gnupg.useLegacyGpg=true
 # buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy.
 # To build a custom Beam version make sure you change it in both places, see
 # https://github.com/apache/beam/issues/21302.
-version=2.53.0-SNAPSHOT
-sdk_version=2.53.0.dev
+version=2.54.0-SNAPSHOT
+sdk_version=2.54.0.dev
 
 javaVersion=1.8
 
diff --git a/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java b/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java
index d249d43d3789..e53de5bdb678 100644
--- a/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java
+++ b/it/common/src/main/java/org/apache/beam/it/common/utils/PipelineUtils.java
@@ -22,6 +22,8 @@
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -115,7 +117,13 @@ public static String createJobName(String prefix, int randomChars) {
 
   /** Get raw job name (without prefix) from a jobName generated by createJobName. */
   public static String extractJobName(String nameWithPrefix) {
-    return nameWithPrefix.substring(0, nameWithPrefix.lastIndexOf("-"));
+    Pattern pattern = Pattern.compile("-\\d{17}"); // yyyyMMddHHmmssSSS
+    Matcher matcher = pattern.matcher(nameWithPrefix);
+    if (matcher.find()) {
+      return nameWithPrefix.substring(0, matcher.start());
+    } else {
+      throw new IllegalArgumentException("Unexpected job name: " + nameWithPrefix);
+    }
   }
 
   /*
diff --git a/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java b/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java
index 316283cdf7d2..d2b8334964f5 100644
--- a/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java
+++ b/it/common/src/test/java/org/apache/beam/it/common/utils/PipelineUtilsTest.java
@@ -51,4 +51,10 @@ public void testCreateExtractJobName() {
     String name = "create-job-name";
     assertEquals(name, extractJobName(createJobName(name)));
   }
+
+  @Test
+  public void testCreateExtractJobNameWithRandomChars() {
+    String name = "create-job-name";
+    assertEquals(name, extractJobName(createJobName(name, 8)));
+  }
 }
diff --git a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java
index 620d24d4e117..e0f9f6c2f63d 100644
--- a/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java
+++ b/it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/dataflow/DefaultPipelineLauncher.java
@@ -410,9 +410,9 @@ protected LaunchInfo.Builder getJobInfoBuilder(LaunchConfig options, JobState st
     // config pipelineName
     String pipelineName = PipelineUtils.extractJobName(options.jobName());
     String overrideName = null;
-    if (pipelineName.endsWith("write")) {
+    if (pipelineName.startsWith("write")) {
       overrideName = System.getProperty(WRITE_PIPELINE_NAME_OVERWRITE);
-    } else if (pipelineName.endsWith("read")) {
+    } else if (pipelineName.startsWith("read")) {
       overrideName = System.getProperty(READ_PIPELINE_NAME_OVERWRITE);
     }
     if (!Strings.isNullOrEmpty(overrideName)) {
diff --git a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
index 4589f79f1aaa..214d2745f03e 100644
--- a/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
+++ b/it/google-cloud-platform/src/test/java/org/apache/beam/it/gcp/bigquery/BigQueryStreamingLT.java
@@ -428,14 +428,14 @@ public CrashingBigQueryServices(Integer crashIntervalSeconds) {
     }
 
     @Override
-    public DatasetService getDatasetService(BigQueryOptions options) {
-      return new CrashingDatasetService(options);
+    public WriteStreamService getWriteStreamService(BigQueryOptions options) {
+      return new CrashingWriteStreamService(options);
     }
 
-    private class CrashingDatasetService extends BigQueryServicesImpl.DatasetServiceImpl {
+    private class CrashingWriteStreamService extends BigQueryServicesImpl.WriteStreamServiceImpl {
       private Instant lastCrash;
 
-      public CrashingDatasetService(BigQueryOptions bqOptions) {
+      public CrashingWriteStreamService(BigQueryOptions bqOptions) {
         super(bqOptions);
       }
 
diff --git a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java
index 6d50dddb0ccd..73c55d001381 100644
--- a/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java
+++ b/it/jdbc/src/main/java/org/apache/beam/it/jdbc/AbstractJDBCResourceManager.java
@@ -99,11 +99,15 @@ public String getPassword() {
     return password;
   }
 
+  public int getPort() {
+    return this.getPort(getJDBCPort());
+  }
+
   @Override
   public synchronized String getUri() {
     return String.format(
         "jdbc:%s://%s:%d/%s",
-        getJDBCPrefix(), this.getHost(), this.getPort(getJDBCPort()), this.getDatabaseName());
+        getJDBCPrefix(), this.getHost(), this.getPort(), this.getDatabaseName());
   }
 
   public abstract String getJDBCPrefix();
diff --git a/learning/prompts/code-explaination/02_io_pubsub.md b/learning/prompts/code-explaination/02_io_pubsub.md
new file mode 100644
index 000000000000..de02bd58e4d8
--- /dev/null
+++ b/learning/prompts/code-explaination/02_io_pubsub.md
@@ -0,0 +1,33 @@
+Prompt:
+What does this code do?
+```python
+pipeline_options = PipelineOptions(pipeline_args)
+
+with beam.Pipeline(options=options) as p:
+    (p | "Read" >> ReadFromPubSub(subscription=input_subscription)
+       | "Process" >> ParDo(ProcessMessage())
+       | "Write" >> WriteToPubSub(topic=output_topic))
+```
+Response:
+This code reads messages from the [Pub/Sub](https://cloud.google.com/pubsub/docs/overview) subscription `input_subscription` using the `ReadFromPubSub`, processes them. The code then writes the resulting collection to the Pub/Sub topic `output_topic` using the `WriteToPubSub` transform.
+
+Pub/Sub is only supported in streaming pipelines.
+
+Reading messages directly from a topic is also supported. In this case, a temporary subscription will be created automatically.
+
+`ReadFromPubSub` produces a `PCollection` of `PubsubMessage` objects or a `PCollection` of byte sequences. Behavior is controlled by the `with_attributes` parameter, with byte sequences being the default.
+For more information about PCollections see the [Beam Programming Guide](https://beam.apache.org/documentation/basics/#pcollection).
+
+The `ProcessMessage` class processes the messages. This class is a subclass of the `DoFn` class.
+The simplest implementation of `ProcessMesage` class might look like the following example:
+
+```python
+class ProcessMessage(beam.DoFn):
+    def process(self, element):
+        yield element
+```
+For more information about the `DoFn` class, see the [Beam Programming Guide](https://beam.apache.org/documentation/programming-guide/#dofn).
+
+For more details, see the [Pub/Sub I/O transform documentation](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.pubsub.html).
+
+For a common pattern for configuring pipeline options, see the [Pipeline option pattern](https://beam.apache.org/documentation/patterns/pipeline-options/).
\ No newline at end of file
diff --git a/learning/prompts/code-generation/02_io_pubsub.md b/learning/prompts/code-generation/02_io_pubsub.md
new file mode 100644
index 000000000000..90856e8956ff
--- /dev/null
+++ b/learning/prompts/code-generation/02_io_pubsub.md
@@ -0,0 +1,51 @@
+Prompt:
+Write the python code to read messages from a Pub/Sub subscription.
+Response:
+You can read messages from a Pub/Sub subscription or topic using the `ReadFromPubSub` transform. Pub/Sub is only supported in streaming pipelines.
+
+The following Python code reads messages from a Pub/Sub subscription. The subscription is provided as a command line argument. The messages are logged to the console:
+
+```python
+import logging
+
+import apache_beam as beam
+from apache_beam import Map
+from apache_beam.io import ReadFromPubSub
+from apache_beam.options.pipeline_options import PipelineOptions
+
+class PubSubReadOptions(PipelineOptions):
+"""
+Configure pipeline options for PubSub read transform.
+"""
+    @classmethod
+    def _add_argparse_args(cls, parser):
+      parser.add_argument(
+          "--subscription",
+          required=True,
+          help="Pub/Sub subscription to read from.")
+
+def read_subscription():
+  """Read from Pub/Sub subscription function."""
+
+  # Parse the pipeline options.
+  # For streaming pipelines, you need to include streaming=True
+  options = PubSubReadOptions(streaming=True)
+
+  with beam.Pipeline(options=options) as p:
+    # This pipeline reads from a Pub/Sub subscription and logs the messages to the console.
+    (p | "Read Pub/Sub subscription" >> ReadFromPubSub(subscription=options.subscription)
+       | "Format message" >> Map(lambda message: f"Received message:\n{message}\n")
+       | Map(logging.info))
+
+if __name__ == '__main__':
+  logging.getLogger().setLevel(logging.INFO)
+  read_subscription()
+
+```
+Reading messages directly from a topic is also supported. A temporary subscription will be created automatically.
+
+The messages could be returned as a byte string or as `PubsubMessage` objects. This behavior is controlled by the `with_attributes` parameter.
+
+For more information, see the [Pub/Sub I/O transform documentation](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.pubsub.html).
+
+For a common pattern for configuring pipeline options see the [Pipeline option pattern](https://beam.apache.org/documentation/patterns/pipeline-options/).
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/01_basic_learning_apache_beam.md b/learning/prompts/documentation-lookup/01_basic_learning_apache_beam.md
new file mode 100644
index 000000000000..df428d910388
--- /dev/null
+++ b/learning/prompts/documentation-lookup/01_basic_learning_apache_beam.md
@@ -0,0 +1,14 @@
+Prompt:
+What is Apache Beam?
+Response:
+Apache Beam is an open-source unified programming model for data processing pipelines. It provides a set of language-specific SDKs that allow you to define and execute data processing workflows.
+Apache Beam allows you to write pipelines in one of the supported languages (Java, Python, Go, and Scala). The pipelines can be executed on one of the supported runners (Apache Flink, Apache Spark, and Google Cloud Dataflow).
+Learning Apache Beam is a valuable skill for data processing and analysis. Here are some resources to get you started:
+[Get Started](https://beam.apache.org/get-started/) section of the Apache Beam website is a perfect place to start. It provides an overview of Apache Beam and its key concepts.
+For a hands-on introduction, you can start by exploring the [WordCount example](https://beam.apache.org/get-started/wordcount-example/), which demonstrates the basic concepts of Apache Beam.
+The [Tour of Beam](https://tour.beam.apache.org/) is an interactive tour that teaches you core Beam concepts. It provides a sandbox environment to write and run pipelines while exploring different topics.
+The [Beam Playground](https://play.beam.apache.org/) offers an interactive space to experiment with Beam's transforms and examples without installing anything on your system. After you have a basic understanding of Apache Beam, you can start to build your own pipelines.
+The Apache Beam website has a number of resources that can help you get started, including [Apache Beam Documentation](https://beam.apache.org/documentation/), and [Apache Beam Community](https://beam.apache.org/community/).
+For those interested in the source code, you can explore the [Apache Beam repository](https://github.com/apache/beam) on GitHub.
+By leveraging these resources, you can build a strong foundation in Apache Beam and start constructing your own data processing pipelines.
+Additionally, Apache Beam is an open-source project, and contributions are welcome. Whether you're fixing bugs, improving documentation, or adding new features, your work can benefit the entire Beam community. To learn more about how you can contribute, visit the [Contribute](https://beam.apache.org/contribute/#) section of the Apache Beam website.
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/02_basic_pipelines.md b/learning/prompts/documentation-lookup/02_basic_pipelines.md
new file mode 100644
index 000000000000..873329b821b7
--- /dev/null
+++ b/learning/prompts/documentation-lookup/02_basic_pipelines.md
@@ -0,0 +1,19 @@
+Prompt:
+What is a Pipeline in Apache Beam?
+Response:
+In Apache Beam, a [pipeline](https://beam.apache.org/documentation/pipelines/design-your-pipeline/) encapsulates the entirety of a data processing task, including all the data and each step of the process. Essentially, it's a [Directed Acyclic Graph](https://en.wikipedia.org/wiki/Directed_acyclic_graph) of transformations (known as `PTransform`) applied to data collections (`PCollection`).
+
+The simplest pipelines in Apache Beam follow a linear flow of operations, typically adhering to a read-process-write pattern. However, pipelines can also be significantly more complex, featuring multiple input sources, output sinks, and operations (known as a `PTransform`) that can both read from and output to multiple PCollections.
+For more information about pipeline design and best practices, see [Common Pipeline Patterns](https://beam.apache.org/documentation/patterns/overview/).
+
+To use Beam, your driver program must first create an instance of the Beam SDK class `Pipeline` (typically in the `main()` function).
+
+```python
+import apache_beam as beam
+
+with beam.Pipeline() as pipeline:
+  pass  # Build your pipeline here.
+```
+
+When you create your pipeline, you need to set [configuration options](https://beam.apache.org/documentation/programming-guide/#configuring-pipeline-options). You can set your pipeline’s configuration options programmatically. You can also set the options ahead of time, or read them from the command line, and then pass them to the `Pipeline` object when you create the object. This option is often easier than setting the pipeline configuration options programmatically.
+
diff --git a/learning/prompts/documentation-lookup/03_basic_configuring_pipelines.md b/learning/prompts/documentation-lookup/03_basic_configuring_pipelines.md
new file mode 100644
index 000000000000..bf5ac7fabf81
--- /dev/null
+++ b/learning/prompts/documentation-lookup/03_basic_configuring_pipelines.md
@@ -0,0 +1,29 @@
+Prompt:
+How do I configure pipeline in Apache Beam?
+Response:
+Configuring pipeline options in Apache Beam is crucial for specifying the execution environment, managing resources, and tailoring the pipeline's behavior to meet specific data processing needs and performance requirements.
+
+You can set pipeline options programmatically or pass them in using the command line. These options include runner-specific settings, job name, project ID (for cloud runners), machine types, number of workers, and more.
+
+Apache Beam offers a variety of [standard pipeline options](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/options/pipeline_options.py) that allow you to customize and optimize your data processing pipelines.
+
+Beam SDKs include a command-line parser that you can use to set pipeline options. Use command-line arguments in the format `--<option>=<value>`. For example, the following command sets the `--runner` option `DirectRunner` and the `--project` option `my-project-id`:
+
+```bash
+python my-pipeline.py --runner=DirectRunner --project=my-project-id
+```
+
+To set the pipeline options programmatically, use the `PipelineOptions` class. For example, the following code sets the `--runner` option to `DirectRunner` and the `--project` option to `my-project-id`:
+
+```python
+from apache_beam import Pipeline
+from apache_beam.options.pipeline_options import PipelineOptions
+
+options = PipelineOptions(
+    project='my-project-id',
+    runner='DirectRunner'
+)
+``````
+In addition to the standard pipeline options, you can add custom pipeline options. For a common pattern for configuring pipeline options, see [Pipeline option pattern](https://beam.apache.org/documentation/patterns/pipeline-options/).
+
+The [WordCount example pipeline](https://beam.apache.org/get-started/wordcount-example/#using-parameterizable-pipelineoptions) demonstrates how to set pipeline options at runtime by using command-line options.
diff --git a/learning/prompts/documentation-lookup/04_basic_pcollections.md b/learning/prompts/documentation-lookup/04_basic_pcollections.md
new file mode 100644
index 000000000000..058425e4f569
--- /dev/null
+++ b/learning/prompts/documentation-lookup/04_basic_pcollections.md
@@ -0,0 +1,25 @@
+Prompt:
+What is a PCollection in Apache Beam?
+Response:
+
+A `PCollection` in Apache Beam is an an unordered bag of elements. Each `PCollection` is a potentially distributed, homogeneous data set or data stream, and is owned by the specific `Pipeline` object for which it is created. It’s the primary data structure used in Apache Beam pipelines to handle large-scale data processing in batch and streaming modes.
+
+The following example shows how to create a `PCollection` using the `Create` transform:
+
+```python
+import apache_beam as beam
+
+with beam.Pipeline() as pipeline:
+  pcollection = pipeline | beam.Create([...])  # Create a PCollection
+```
+
+A `PCollection` can either be bounded or unbounded, making it versatile for different [data source types](https://beam.apache.org/documentation/basics/#pcollection). Bounded `PCollections` represent a finite data set, such as files or databases, ideal for batch processing. Unbounded `PCollections`, on the other hand, represent data streams that continuously grow over time, such as real-time event logs, suitable for stream processing.
+
+Beam’s computational patterns and transforms are focused on situations where distributed data-parallel computation is required. Therefore, a `PCollection` has the following key characteristics:
+   - All elements must be of the same type (with support of structured types).
+   - Every `PCollection` has a coder, which is a specification of the binary format of the elements.
+   - Elements are immutable. They can't be altered after creation.
+   - Random access to individual elements of the collection isn't allowed.
+   - Beam encodes each element for distributed processing.
+
+For more information about `Pcollection` objects in Apache Beam and for usage patterns, see [Beam Programming Guide: PCollections](https://beam.apache.org/documentation/programming-guide/#pcollections).
diff --git a/learning/prompts/documentation-lookup/05_basic_ptransforms.md b/learning/prompts/documentation-lookup/05_basic_ptransforms.md
new file mode 100644
index 000000000000..fcdc6481d694
--- /dev/null
+++ b/learning/prompts/documentation-lookup/05_basic_ptransforms.md
@@ -0,0 +1,34 @@
+Prompt:
+What is a PTransform in Apache Beam?
+Response:
+
+A [`PTransform`](https://beam.apache.org/documentation/programming-guide/#transforms) (or transform) represents a data processing operation, or a step, in a Beam pipeline. A transform is applied to zero or more `PCollection` objects and produces zero or more `PCollection` objects.
+
+Transforms have the following key characteristics:
+1. Versatility: Able to execute a diverse range of operations on `PCollection` objects.
+2. Composability: Can be combined to form elaborate data processing pipelines.
+3. Parallel execution: Designed for distributed processing, allowing simultaneous execution across multiple workers.
+4. Scalability: Able to handle extensive data and suitable for both batch and streaming data.
+
+The Beam SDKs contain different transforms that you can apply to your pipeline’s `PCollection` objects. The following list includes common transform types:
+ - [Source transforms](https://beam.apache.org/documentation/programming-guide/#pipeline-io) such as `TextIO.Read` and `Create`. A source transform conceptually has no input.
+ - [Processing and conversion operations](https://beam.apache.org/documentation/programming-guide/#core-beam-transforms) such as `ParDo`, `GroupByKey`, `CoGroupByKey`, `Combine`, and `Count`.
+ - [Outputting transforms](https://beam.apache.org/documentation/programming-guide/#pipeline-io) such as `TextIO.Write`.
+ - User-defined, application-specific [composite transforms](https://beam.apache.org/documentation/programming-guide/#composite-transforms).
+
+Transform processing logic is provided in the form of a function object, colloquially referred to as “user code.” This code is applied to each element of the input `PCollection` (or more than one `PCollection`). The `PCollection` objects can be linked together to create complex data processing sequences.
+User code for transforms must satisfy the [requirements of the Beam model](https://beam.apache.org/documentation/programming-guide/#requirements-for-writing-user-code-for-beam-transforms).
+
+The following example shows how to apply custom user code to a `PCollection` using the `ParDo` transform:
+
+```python
+import apache_beam as beam
+
+def SomeUserCode(element):
+    # Do something with element
+    return element
+
+with beam.Pipeline() as pipeline:
+    input_collection = pipeline | beam.Create([...])
+    output_collection = input_collection | beam.ParDo(SomeUserCode())
+```
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/06_basic_schema.md b/learning/prompts/documentation-lookup/06_basic_schema.md
new file mode 100644
index 000000000000..cecbba174576
--- /dev/null
+++ b/learning/prompts/documentation-lookup/06_basic_schema.md
@@ -0,0 +1,25 @@
+Prompt:
+What are schemas in Apache Beam
+Response:
+
+ A [Schema in Apache Beam](https://beam.apache.org/documentation/programming-guide/#schemas) is a language-independent type definition for a `PCollection`. The schema defines elements of that `PCollection` as an ordered list of named fields.
+
+In many cases, the element type in a `PCollection` has a structure that can be introspected. Some examples are JSON, Protocol Buffer, Avro, and database row objects. All of these formats can be converted to Beam Schemas.
+
+To take advantage of schemas, your `PCollection` objects must have a schema attached. Often, the source itself attaches a schema to the `PCollection`.
+
+Although schemas are language-independent, they're designed to embed naturally into the Beam SDK programming languages. This feature lets you continue [to use native types](https://beam.apache.org/documentation/programming-guide/#schemas-for-pl-types) while leveraging the advantage of having Beam understand your element schemas.
+
+Beam provides a collection of [schema transforms](https://beam.apache.org/documentation/programming-guide/#662-schema-transforms) that operate on schemas. For an example, see [SqlTransform](https://beam.apache.org/documentation/dsls/sql/overview/).
+
+The following list includes key capabilities provided by schema transforms:
+* field selection
+* grouping and aggregation
+* join operations
+* filtering data
+* adding and removing fields
+* renaming fields
+* type conversion
+* enhanced ParDo functionality
+
+For more information about schemas and for Beam best practices when working with structured data, see [Schema Usage Patterns](https://beam.apache.org/documentation/patterns/schema/).
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/07_basic_runners.md b/learning/prompts/documentation-lookup/07_basic_runners.md
new file mode 100644
index 000000000000..982c06a44151
--- /dev/null
+++ b/learning/prompts/documentation-lookup/07_basic_runners.md
@@ -0,0 +1,23 @@
+Prompt:
+What is a Runner in Apache Beam?
+Response:
+Apache Beam Runners are the execution engines that run the pipelines. They translate or adapt the pipeline into a form that can be executed on a massively parallel big data processing system, such as Apache Flink, Apache Spark, Google Cloud Dataflow, and more.
+
+Choosing a runner is an important step in the pipeline development process. The runner you choose determines where and how your pipeline runs. For more information about available runners and their capabilities, see the [capabilities matrix](https://beam.apache.org/documentation/runners/capability-matrix/).
+
+To specify a runner, when you execute the pipeline, use the `--runner` flag . For example, to run the WordCount pipeline on Google Cloud Dataflow, you would run the following command:
+
+```python
+python -m apache_beam.examples.wordcount \
+  --region DATAFLOW_REGION \
+  --input gs://dataflow-samples/shakespeare/kinglear.txt \
+  --output gs://STORAGE_BUCKET/results/outputs \
+  --runner DataflowRunner \
+  --project PROJECT_ID \
+  --temp_location gs://STORAGE_BUCKET/tmp/
+  ```
+For more details about the WordCount example, see the [WordCount source code in GitHub](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/wordcount.py).
+
+The [Direct Runner](https://beam.apache.org/documentation/runners/direct/) executes the pipeline on your local machine. It is helpful for testing and debugging purposes.
+
+For instructions about how to configure your development environment and runner, see the quickstarts for [Java](https://beam.apache.org/get-started/quickstart-java), [Python](https://beam.apache.org/get-started/quickstart-py) and [Go](https://beam.apache.org/get-started/quickstart-go).
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/08_basic_windowing.md b/learning/prompts/documentation-lookup/08_basic_windowing.md
new file mode 100644
index 000000000000..7a9d7a100d21
--- /dev/null
+++ b/learning/prompts/documentation-lookup/08_basic_windowing.md
@@ -0,0 +1,29 @@
+Prompt:
+What is Windowing in Apache Beam?
+Response:
+In stream processing, windowing lets you to divide data streams into logical units for efficient and correct parallel processing.
+
+With an unbounded data set, collecting all of the elements is impossible, because new elements are constantly being added. In the Beam model, any `PCollection` (including unbounded `PCollection` objects) can be subdivided into [logical windows](https://beam.apache.org/documentation/programming-guide/#windowing-basics). Grouping transforms then consider the elements of each `PCollection` on a per-window basis.
+
+Because Beam's default windowing strategy is to assign each element to a single, global window, you must explicitly specify a [windowing function](https://beam.apache.org/documentation/programming-guide/#setting-your-pcollections-windowing-function) for your pipeline.
+
+The following code snippet shows how  to divide a `PCollection` into 60-second windows:
+```python
+from apache_beam import beam
+from apache_beam import window
+fixed_windowed_items = (
+    items | 'window' >> beam.WindowInto(window.FixedWindows(60)))
+```
+
+Beam provides a number of [built-in windowing functions](https://beam.apache.org/documentation/programming-guide/#provided-windowing-functions) that you can use to subdivide your data into different types of windows:
+- Fixed Time Windows
+- Sliding Time Windows
+- Per-Session Windows
+- Single Global Window
+- Calendar-based Windows (not supported by the Beam SDK for Python or Go)
+
+You can also create your own custom windowing function [WindowFn](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/transforms/window.py).
+
+You also need to specify a [triggering strategy](https://beam.apache.org/documentation/programming-guide/#triggers) to determine when to emit the results of your pipeline’s windowed computations.
+
+You can adjust the windowing strategy to allow for [late data](https://beam.apache.org/documentation/programming-guide/#watermarks-and-late-data), which is data that arrives after the watermark has passed the end of the window. You can also specify how to handle late data, such as discarding or adding it to the next window.
diff --git a/learning/prompts/documentation-lookup/09_basic_triggers.md b/learning/prompts/documentation-lookup/09_basic_triggers.md
new file mode 100644
index 000000000000..69d7b7bd9145
--- /dev/null
+++ b/learning/prompts/documentation-lookup/09_basic_triggers.md
@@ -0,0 +1,32 @@
+Prompt:
+What are Triggers in Apache Beam?
+Response:
+Beam uses triggers to determine when to emit the aggregated results of each [window](https://beam.apache.org/documentation/programming-guide/#windowing), referred to as a pane.
+
+Triggers provide two additional capabilities compared to [outputting at the end of a window](https://beam.apache.org/documentation/programming-guide/#default-trigger):
+
+- Allowing early results to be output before the end of the window
+- Allowing late data to be handled after the end of the window
+
+This feature lets you control the flow of your data and balance between completeness, latency, and cost.
+
+You set the triggers for a `PCollection` by setting the `trigger` parameter of the `WindowInto` transform.
+
+```python
+  pcollection | WindowInto(
+    FixedWindows(1 * 60),
+    trigger=AfterProcessingTime(1 * 60),
+    accumulation_mode=AccumulationMode.DISCARDING)
+```
+
+When a trigger fires, it emits the current contents of the window as a pane. Because a trigger can fire multiple times, the accumulation mode determines whether the system accumulates the window panes as the trigger fires, or discards them. This behavior is controlled by the [window accumulation mode](https://beam.apache.org/documentation/programming-guide/#window-accumulation-modes) parameter of the `WindowInto` transform.
+
+
+Beam provides several [built-in triggers](https://beam.apache.org/documentation/programming-guide/#triggers) that you can use to determine when to emit the results of your pipeline's windowed computations:
+- Event time triggers
+- Processing time triggers
+- Data-driven triggers
+- Composite triggers
+
+One of the most useful trigger patterns is the [AfterWatermark](https://beam.apache.org/documentation/programming-guide/#composite-afterwatermark) trigger, which fires a single time when Beam estimates that all the data has arrived, such as when the watermark passes the end of the window.
+
diff --git a/learning/prompts/documentation-lookup/10_basic_metrics.md b/learning/prompts/documentation-lookup/10_basic_metrics.md
new file mode 100644
index 000000000000..5899819acbf0
--- /dev/null
+++ b/learning/prompts/documentation-lookup/10_basic_metrics.md
@@ -0,0 +1,24 @@
+Prompt:
+What are Metrics in Apache Beam?
+Response:
+[Metrics](https://beam.apache.org/documentation/programming-guide/#metrics) in Apache Beam provides a way to get insight into the current state of your pipeline, including during pipeline execution.
+
+Metrics are named and scoped to a specific step in the pipeline. They can be created dynamically during pipeline execution. If a runner doesn't support some part of reporting metrics, the fallback behavior is to drop the metric updates rather than fail the pipeline.
+
+Beam provides a number of [built-in metric types](https://beam.apache.org/documentation/programming-guide/#types-of-metrics):
+* Counters
+* Distributions
+* Gauges
+
+To declare a metric, use the `beam.metrics.Metrics` class. For example:
+```python
+self.words_counter = Metrics.counter(self.__class__, 'words')
+self.word_lengths_counter = Metrics.counter(self.__class__, 'word_lengths')
+self.word_lengths_dist = Metrics.distribution(self.__class__, 'word_len_dist')
+self.empty_line_counter = Metrics.counter(self.__class__, 'empty_lines')
+```
+
+For implementation details, see the [WordCount example with metrics](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/examples/wordcount_with_metrics.py).
+
+
+You can export metrics to external sinks. Spark and Flink runners support REST HTTP and Graphite.
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/11_basic_state.md b/learning/prompts/documentation-lookup/11_basic_state.md
new file mode 100644
index 000000000000..a4bf757a11b4
--- /dev/null
+++ b/learning/prompts/documentation-lookup/11_basic_state.md
@@ -0,0 +1,20 @@
+Prompt:
+What is a State in Apache Beam?
+Response:
+[State](https://beam.apache.org/documentation/basics/#state-and-timers) in Apache Beam is an API that allows for managing per-key state in element-wise operations like [ParDo](https://beam.apache.org/documentation/programming-guide/#pardo) or Map. It enables fine-grained control over aggregations, particularly in situations where [windowing](https://beam.apache.org/documentation/programming-guide/#windowing) and [triggers](https://beam.apache.org/documentation/programming-guide/#triggers) are insufficient.
+
+State is [scoped](https://beam.apache.org/documentation/programming-guide/#state-and-timers) to the current processing key within a window, meaning each key has its own state. State is particularly useful for complex aggregations across multiple elements.
+
+Beam provides several [types of state](https://beam.apache.org/documentation/programming-guide/#types-of-state) that can be used for different use cases, including scalar and collection values:
+- ValueState
+- BagState
+- SetState
+- MapState
+- OrderedListState
+- CombiningState
+
+The per-key state needs to be garbage collected, because the increasing size of the state can negatively impact performance. For more information about common garbage collection strategies, see [Garbage collecting state](https://beam.apache.org/documentation/programming-guide/#garbage-collecting-state) .
+
+For more information, read the community blog post about [Stateful Processing](https://beam.apache.org/blog/stateful-processing/).
+
+For additional examples of using State in Apache Beam see [State and Time examples](https://beam.apache.org/documentation/programming-guide/#state-timers-examples).
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/12_basic_timers.md b/learning/prompts/documentation-lookup/12_basic_timers.md
new file mode 100644
index 000000000000..e947c66365f8
--- /dev/null
+++ b/learning/prompts/documentation-lookup/12_basic_timers.md
@@ -0,0 +1,12 @@
+Prompt:
+What is a Timer in Apache Beam?
+Response:
+In Apache Beam, a [Timer](https://beam.apache.org/documentation/basics/#state-and-timers) is a per-key timer callback API that enables delayed processing of data stored using the [State API](https://beam.apache.org/documentation/programming-guide/#state-and-timers)
+
+Apache Beam provides two [types of timers](https://beam.apache.org/documentation/programming-guide/#timers): processing time timers and event time timers. Processing time timers are based on the system clock. Event time timers are based on the timestamps of the data elements.
+
+Beam also supports dynamically setting a timer tag using `TimerMap` in the Java SDK. This feature lets you set multiple different timers in a `DoFn`. You can set the timer tags to be chosen dynamically chosen, for example, based on data in the input elements.
+
+For more information, see the community blog post about [Timely Processing](https://beam.apache.org/blog/timely-processing/).
+
+For additional examples of using timers in Apache Beam see [State and Time examples](https://beam.apache.org/documentation/programming-guide/#state-timers-examples).
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/13_advanced_splittable_dofn.md b/learning/prompts/documentation-lookup/13_advanced_splittable_dofn.md
new file mode 100644
index 000000000000..41822a931fbf
--- /dev/null
+++ b/learning/prompts/documentation-lookup/13_advanced_splittable_dofn.md
@@ -0,0 +1,18 @@
+Prompt:
+What is Splittable DoFn in Apache Beam?
+Response:
+A splittable `DoFn` (SDF) is a generalization of [`DoFn`](https://beam.apache.org/documentation/programming-guide/#pardo) that lets you process elements in a non-monolithic way. A splittable `DoFn` makes it easier to create complex, modular I/O connectors in Beam .
+When you apply a splittable `DoFn` to an element, the runner can split the element’s processing into smaller tasks. You can checkpoint the processing of an element, and you can split the remaining work to yield additional parallelism.
+
+At a high level, an SDF is responsible for processing element and restriction pairs. A restriction represents a subset of work that would have been necessary to have been done when processing the element.
+
+Executing a [Splittable `DoFn`](https://beam.apache.org/documentation/programming-guide/#splittable-dofns) uses the following steps:
+1. Each element is paired with a restriction. For example, filename is paired with offset range representing the whole file).
+2. Each element and restriction pair is split. For example, offset ranges are broken up into smaller pieces.
+3. The runner redistributes the element and restriction pairs to several workers.
+4. Element and restriction pairs are processed in parallel. For example, the file is read. Within this last step, the element and restriction pair can pause its own processing or be split into further element and restriction pairs.
+
+For an example, see the [Splittable DoFn module](https://tour.beam.apache.org/tour/python/splittable-dofn/splittable) in the Tour of Beam.
+
+For more information, see the [community blog post](https://beam.apache.org/blog/splittable-do-fn-is-available/).
+
diff --git a/learning/prompts/documentation-lookup/14_advanced_pipeline_patterns.md b/learning/prompts/documentation-lookup/14_advanced_pipeline_patterns.md
new file mode 100644
index 000000000000..f8cfaa86accd
--- /dev/null
+++ b/learning/prompts/documentation-lookup/14_advanced_pipeline_patterns.md
@@ -0,0 +1,13 @@
+Prompt:
+What pipeline patterns exist in Apache Beam?
+Response:
+Beam pipeline patterns are a set of best practices for building Beam pipelines. They are based on real-world Beam deployments and are designed to help you build resilient, flexible, and portable Beam pipelines.
+
+The following list includes some of the most common pipeline patterns:
+- [File processing patterns](https://beam.apache.org/documentation/patterns/file-processing/)
+- [Pipeline options patterns](https://beam.apache.org/documentation/patterns/pipeline-options/)
+- [Side input patterns](https://beam.apache.org/documentation/patterns/side-inputs/)
+- [Windowing patterns](https://beam.apache.org/documentation/patterns/custom-windows/)
+- [State and Timers patterns](https://beam.apache.org/documentation/patterns/grouping-elements-for-efficient-external-service-calls/)
+
+For more information, see the [Pipeline Patterns](https://beam.apache.org/documentation/patterns/overview/) section of the Beam programming guide.
diff --git a/learning/prompts/documentation-lookup/15_advanced_xlang.md b/learning/prompts/documentation-lookup/15_advanced_xlang.md
new file mode 100644
index 000000000000..f7df6785fabd
--- /dev/null
+++ b/learning/prompts/documentation-lookup/15_advanced_xlang.md
@@ -0,0 +1,15 @@
+Prompt:
+What is a multi-language pipeline in Apache Beam?
+Response:
+Beam lets you combine transforms written in any supported SDK language (currently, [Java](https://beam.apache.org/documentation/programming-guide/#1311-creating-cross-language-java-transforms) and [Python](https://beam.apache.org/documentation/programming-guide/#1312-creating-cross-language-python-transforms)) and use them in one multi-language pipeline. For example, a multi-language pipeline might read from a Python source, process the data using a Java transform, and write the data to a Python sink.
+
+For example, you can use the [Apache Kafka connector](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/io/kafka.py) and [SQL transform](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/transforms/sql.py) from the Java SDK in Python pipelines.
+
+To learn how to create a multi-language pipeline, see the quickstart examples for [Java](https://beam.apache.org/documentation/sdks/java-multi-language-pipelines) and [Python](https://beam.apache.org/documentation/sdks/python-multi-language-pipelines).
+
+Depending on the SDK language of the pipeline, you can use a high-level SDK-wrapper class or a low-level transform class to access a cross-language transform. For more information, see [Using cross-language transforms](https://beam.apache.org/documentation/programming-guide/#use-x-lang-transforms).
+
+To develop a cross-language transform, you need to define a Uniform Resourse Name(URN) for registering the transform with an expansion service. For more information, see [Defining a URN](https://beam.apache.org/documentation/programming-guide/#1314-defining-a-urn).
+
+
+
diff --git a/learning/prompts/documentation-lookup/16_advanced_pipeline_lifecycle.md b/learning/prompts/documentation-lookup/16_advanced_pipeline_lifecycle.md
new file mode 100644
index 000000000000..c59a177c2525
--- /dev/null
+++ b/learning/prompts/documentation-lookup/16_advanced_pipeline_lifecycle.md
@@ -0,0 +1,36 @@
+Prompt:
+What is a pipeline development lifecycle in Apache Beam?
+Response:
+
+The Apache Beam pipeline development lifecycle is an iterative process that usually involves the following steps:
+
+- Design your pipeline.
+- Develop your pipeline code.
+- Test your pipeline.
+- Deploy your pipeline.
+
+During each iteration, you might need to go back and forth between the different steps to refine your pipeline code and to fix bugs.
+
+To design a pipeline, you need answers to the following questions:
+- Where is the data stored?
+- What does the data look like?
+- What do you want to do with your data?
+- What does your output data look like, and where should it go?
+
+Beam documentation has more information about [pipeline design](https://beam.apache.org/documentation/pipelines/design-your-pipeline/) and about common [pipeline patterns](https://beam.apache.org/documentation/patterns/overview/).
+
+
+An Apache Beam program expresses a data processing pipeline, from start to finish. To construct a pipeline using the classes in the Apache Beam SDKs, your program needs to perform the following steps:
+
+- Create a `Pipeline` object.
+- Use a `Read` or `Create` transform to create one or more `PCollection` objects for your pipeline data.
+- Apply transforms to each `PCollection`.
+- Write or otherwise output the final, transformed `PCollection` objects.
+- Run the pipeline.
+
+The Apache Beam documentation has more information about [developing](https://beam.apache.org/documentation/programming-guide/) and [executing](https://beam.apache.org/documentation/pipelines/create-your-pipeline/) pipelines.
+
+Testing pipelines is a particularly important step in developing an effective data processing solution. The indirect nature of the Beam model, in which your user code constructs a pipeline graph to be executed remotely, can make debugging-failed runs difficult. For more information about pipeline testing strategies, see [Test Your Pipeline](https://beam.apache.org/documentation/pipelines/test-your-pipeline/).
+
+Choosing a [runner](https://beam.apache.org/documentation/#choosing-a-runner) is a crucial step in deploying your pipeline. The runner you choose determines where and how your pipeline executes.
+For more information about pipeline deployment, see [Container environments](https://beam.apache.org/documentation/runtime/environments/).
diff --git a/learning/prompts/documentation-lookup/17_advanced_ai_ml.md b/learning/prompts/documentation-lookup/17_advanced_ai_ml.md
new file mode 100644
index 000000000000..afc96ecd5005
--- /dev/null
+++ b/learning/prompts/documentation-lookup/17_advanced_ai_ml.md
@@ -0,0 +1,18 @@
+Prompt:
+What are AI and ML capabilities in Apache Beam?
+Response:
+Apache Beam has several built-in [AI and ML capabilities](https://beam.apache.org/documentation/ml/overview/) that enable you to:
+- Process large datasets for both preprocessing and model inference.
+- Conduct exploratory data analysis and smoothly scale up data pipelines in production as part of your MLOps ecosystem.
+- Run your models in production with varying data loads, both in batch and streaming pipelines.
+
+For common AI platform integration patterns in Apache Beam, see [AI Platform integration patterns](https://beam.apache.org/documentation/patterns/ai-platform/).
+
+The recommended way to implement inference in Apache Beam is by using the [RunInference API](https://beam.apache.org/documentation/sdks/python-machine-learning/). For more information about how to use RunInference for PyTorch, scikit-learn, and TensorFlow, see the [Use RunInference in Apache Beam](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_pytorch_tensorflow_sklearn.ipynb) example in GitHub.
+
+Using pre-trained models in Apache Beam is also supported with [PyTorch](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_pytorch.ipynb), [Scikit-learn](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_sklearn.ipynb), and [Tensorflow](https://github.com/apache/beam/blob/master/examples/notebooks/beam-ml/run_inference_tensorflow.ipynb). Running inference on  [custom models](https://beam.apache.org/documentation/ml/about-ml/#use-custom-models) is also supported.
+
+Apache Beam also supports automatically updating the model being used with the `RunInference PTransform` in streaming pipelines without stopping the pipeline. The feature lets you avoid downtime downtime. For more information, see [Automatic model refresh](https://beam.apache.org/documentation/ml/about-ml/#automatic-model-refresh).
+For more information about using machine learning models with Apache Beam, see [Running ML models now easier with new Dataflow ML innovations on Apache Beam](https://cloud.google.com/blog/products/ai-machine-learning/dataflow-ml-innovations-on-apache-beam/).
+
+For an example that use the Apache Beam ML integration, see [BigQuery ML integration](https://beam.apache.org/documentation/patterns/bqml/).
\ No newline at end of file
diff --git a/learning/prompts/documentation-lookup/19_io_pubsub.md b/learning/prompts/documentation-lookup/19_io_pubsub.md
new file mode 100644
index 000000000000..f2a66dc89008
--- /dev/null
+++ b/learning/prompts/documentation-lookup/19_io_pubsub.md
@@ -0,0 +1,27 @@
+Prompt:
+Is Pub/Sub supported in Apache Beam?
+Response:
+[PubSub](https://cloud.google.com/pubsub) is a[ Google Cloud](https://cloud.google.com/) service that provides a simple, reliable, scalable, and secure real-time messaging service for sending and receiving messages between independent applications. Apache Beam provides a PubSubIO connector that allows you to read and write messages from and to Pub/Sub.
+Pub/Sub is currently supported only in streaming pipelines.
+
+
+Pub/Sub is supported in the following Beam SDKs:
+* [Java](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.html)
+* [Python](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.pubsub.html)
+* [Go](https://pkg.go.dev/github.com/apache/beam/sdks/v2/go/pkg/beam/io/pubsubio)
+* [Typescript](https://github.com/apache/beam/blob/master/sdks/typescript/src/apache_beam/io/pubsub.ts) through X Language.
+
+[Dataflow-cookbook repository](https://github.com/GoogleCloudPlatform/dataflow-cookbook) will help you to get started with Pub/Sub and Apache Beam. See here for [read](https://github.com/GoogleCloudPlatform/dataflow-cookbook/blob/main/Python/pubsub/read_pubsub_multiple.py) and [write](https://github.com/GoogleCloudPlatform/dataflow-cookbook/blob/main/Python/pubsub/write_pubsub.py) examples in Python.
+
+Apache Beam pipeline code for reading data from Pub/Sub topic and writing it back to another topic might look like the following example:
+
+```python
+from apache_beam.io import ReadFromPubSub
+from apache_beam.io import WriteToPubSub
+
+  with beam.Pipeline(options=options) as p:
+    (p | "Read from Pub/Sub" >> ReadFromPubSub(topic="input_topic")
+       | "Write to Pub/Sub" >> WriteToPubSub(topic="output_topic"))
+```
+
+
diff --git a/learning/prompts/documentation-lookup/20_io_biguery.md b/learning/prompts/documentation-lookup/20_io_biguery.md
new file mode 100644
index 000000000000..d4231e0fe336
--- /dev/null
+++ b/learning/prompts/documentation-lookup/20_io_biguery.md
@@ -0,0 +1,39 @@
+Prompt:
+Is BigQuery supported in Apache Beam?
+Response:
+Yes, Apache Beam supports BigQuery. [BigQuery](https://cloud.google.com/bigquery) is a serverless and cost-effective enterprise data warehouse offered by [Google Cloud](https://cloud.google.com/). Apache Beam provides a `BigQueryIO` connector to read and write data to and from BigQuery. The `BigQueryIO` connector supports both batch and streaming pipelines.
+
+The following Apache Beam SDKs support the `BigQueryIO` connector:
+* [Java](https://beam.apache.org/releases/javadoc/current/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.html)
+* [Python](https://beam.apache.org/releases/pydoc/current/apache_beam.io.gcp.bigquery.html)
+* [Go](https://pkg.go.dev/github.com/apache/beam/sdks/v2/go/pkg/beam/io/bigqueryio) native and though [X Language](https://pkg.go.dev/github.com/apache/beam/sdks/v2/go/pkg/beam/io/xlang/bigqueryio)
+* [Typescript](https://github.com/apache/beam/blob/master/sdks/typescript/src/apache_beam/io/bigqueryio.ts) through X Language
+
+To read data from BigQuery, use `ReadFromBigQuery`. Apache Beam can read data directly from a BigQuery table or using a SQL query. The default mode is to return table rows read from a BigQuery source as dictionaries. Built-in `TableRow` objects can also be returned.
+
+Apache Beam pipeline code for reading from BigQuery might look like the following example:
+
+```python
+from apache_beam.io.gcp.bigquery import ReadFromBigQuery
+
+with beam.Pipeline(options=options) as p:
+  # read from a table
+    lines_table = p | 'Read' >> ReadFromBigQuery(table=table)
+  # read from a query
+    lines_query = p | 'Read' >> ReadFromBigQuery(query="SELECT * FROM table")
+
+```
+Apache Beam pipeline code for writing from BigQuery might look like the following example:
+
+```python
+from apache_beam.io.gcp.bigquery import WriteToBigQuery
+
+with beam.Pipeline(options=options) as p:
+  # write to a table
+    p | 'Write' >> beam.io.WriteToBigQuery(
+        table,
+        schema=TABLE_SCHEMA,
+        create_disposition=beam.io.BigQueryDisposition.CREATE_IF_NEEDED,
+        write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND)
+```
+To get started with BigQuery and Apache Beam, see the examples in the [Dataflow-cookbook repository](https://github.com/GoogleCloudPlatform/dataflow-cookbook) in GitHub. For Python read examples, see the [read_table_bigquery.py](https://github.com/GoogleCloudPlatform/dataflow-cookbook/blob/main/Python/bigquery/read_table_bigquery.py) example. For Python write examples, see the [write_bigquery.py](https://github.com/GoogleCloudPlatform/dataflow-cookbook/blob/main/Python/bigquery/write_bigquery.py) example.
diff --git a/learning/tour-of-beam/learning-content/core-transforms/map/co-group-by-key/description.md b/learning/tour-of-beam/learning-content/core-transforms/map/co-group-by-key/description.md
index c0e7f9f3e44d..a15321cd8ea3 100644
--- a/learning/tour-of-beam/learning-content/core-transforms/map/co-group-by-key/description.md
+++ b/learning/tour-of-beam/learning-content/core-transforms/map/co-group-by-key/description.md
@@ -235,7 +235,7 @@ static class ProductWeight {
         private String fruit;
         private Integer productWeight;
 
-        public WordsAlphabet(String country, String fruit, Integer productWeight) {
+        public ProductWeight(String country, String fruit, Integer productWeight) {
             this.country = country;
             this.fruit = fruit;
             this.productWeight = productWeight;
@@ -275,7 +275,7 @@ static PCollection<String> applyTransform(PCollection<String> fruits, PCollectio
                         String fruit = coGbkResult.getOnly(fruitsTag);
                         String country = coGbkResult.getOnly(countriesTag);
 
-                        out.output(new WordsAlphabet(alphabet, fruit, country).toString());
+                        out.output(new ProductWeight(alphabet, fruit, country).toString());
                     }
 
                 }));
@@ -284,42 +284,28 @@ static PCollection<String> applyTransform(PCollection<String> fruits, PCollectio
 {{end}}
 {{if (eq .Sdk "python")}}
 ```
-weight := beam.ParDo(s, func(_ []byte, emit func(string, int)){
-		emit("brazil", 1000)
-		emit("australia", 150)
-		emit("canada", 340)
-}, beam.Impulse(s))
-
-fruits := beam.ParDo(s, func(_ []byte, emit func(string, string)){
-		emit("australia", "cherry")
-		emit("brazil", "apple")
-		emit("canada", "banan")
-}, beam.Impulse(s))
+fruits = p | 'Fruits' >> beam.Create([('australia', 'cherry'), ('brazil', 'apple'), ('canada', 'banana')])
+weights = p | 'Countries' >> beam.Create([('australia', 1000), ('brazil', 150), ('canada', 340)])
 ```
 
-Change `Alphabet` to `ProductWeight`:
+Change `alphabet` to `product_weight`:
 ```
-type WordsAlphabet struct {
-	Country string
-	Fruit string
-	ProductWeight int
-}
+class ProductWeight:
+    def __init__(self, product_weight, fruit, country):
+        self.product_weight = product_weight
+        self.fruit = fruit
+        self.country = country
 ```
 
 The union takes place through the keys:
 ```
-func applyTransform(s beam.Scope, fruits beam.PCollection, countries beam.PCollection) beam.PCollection {
-	grouped := beam.CoGroupByKey(s, fruits, countries)
-	return beam.ParDo(s, func(key string, weightIter func(*int) bool, fruitIter func(*string) bool, emit func(string)) {
-
-	wa := &WordsAlphabet{
-		Country: key,
-	}
-	weightIter(&wa.ProductWeight)
-	fruitIter(&wa.Fruit)
-    emit(wa.String())
-
-	}, grouped)
-}
+def apply_transforms(fruits, weights):
+    def cogbk_result_to_product_weight(cgbk_result):
+        (country, values) = cgbk_result
+        return ProductWeight(values['weights'][0], values['fruits'][0], country)
+
+    return ({'fruits': fruits, 'weights': weights}
+            | beam.CoGroupByKey()
+            | beam.Map(cogbk_result_to_product_weight))
 ```
-{{end}}
\ No newline at end of file
+{{end}}
diff --git a/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto b/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto
index 777ce8636b7b..4b40c7fa4e4e 100644
--- a/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto
+++ b/model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto
@@ -52,7 +52,7 @@ import "google/protobuf/duration.proto";
 message FnApiTransforms {
   enum Runner {
     // DataSource is a Root Transform, and a source of data for downstream
-    // transforms in the same ProcessBundleDescriptor. 
+    // transforms in the same ProcessBundleDescriptor.
     // It represents a stream of values coming in from an external source/over
     // a data channel, typically from the runner. It's not the PCollection itself
     // but a description of how to get the portion of the PCollection for a given
@@ -82,7 +82,7 @@ message FnApiTransforms {
     // request will be sent with the matching instruction ID and transform ID.
     // Each PCollection that exits the ProcessBundleDescriptor subgraph will have
     // it's own DataSink, keyed by a transform ID determined by the runner.
-    //  
+    //
     // The DataSink will take in a stream of elements for a given instruction ID
     // and encode them for transmission to the remote sink. The coder ID must be
     // for a windowed value coder.
@@ -924,6 +924,35 @@ message StateKey {
     bytes window = 3;
   }
 
+  // Represents a request for the keys and values associated with a specified window in a PCollection. See
+  // https://s.apache.org/beam-fn-state-api-and-bundle-processing for further
+  // details.
+  //
+  // This is expected to be more efficient than iterating over they keys and
+  // looking up the values one at a time.  If a runner chooses not to implement
+  // this protocol, or a key has too many values to fit into a single response,
+  // the runner is free to fail the request and a fallback to point lookups
+  // will be performed by the SDK.
+  //
+  // Can only be used to perform StateGetRequests on side inputs of the URN
+  // beam:side_input:multimap:v1.
+  //
+  // For a PCollection<KV<K, V>>, the response data stream will be a
+  // concatenation of all KVs associated with the specified window,
+  // encoded with the the KV<K, Iterable<V>> coder.
+  // See
+  // https://s.apache.org/beam-fn-api-send-and-receive-data for further
+  // details.
+  message MultimapKeysValuesSideInput {
+    // (Required) The id of the PTransform containing a side input.
+    string transform_id = 1;
+    // (Required) The id of the side input.
+    string side_input_id = 2;
+    // (Required) The window (after mapping the currently executing elements
+    // window into the side input windows domain) encoded in a nested context.
+    bytes window = 3;
+  }
+
   // Represents a request for an unordered set of values associated with a
   // specified user key and window for a PTransform. See
   // https://s.apache.org/beam-fn-state-api-and-bundle-processing for further
@@ -999,6 +1028,7 @@ message StateKey {
     BagUserState bag_user_state = 3;
     IterableSideInput iterable_side_input = 4;
     MultimapKeysSideInput multimap_keys_side_input = 5;
+    MultimapKeysValuesSideInput multimap_keys_values_side_input = 8;
     MultimapKeysUserState multimap_keys_user_state = 6;
     MultimapUserState multimap_user_state = 7;
   }
diff --git a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
index e70d978c56ac..a4736f8b4938 100644
--- a/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
+++ b/model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto
@@ -29,6 +29,7 @@ option go_package = "github.com/apache/beam/sdks/v2/go/pkg/beam/model/jobmanagem
 option java_package = "org.apache.beam.model.expansion.v1";
 option java_outer_classname = "ExpansionApi";
 
+import "google/protobuf/struct.proto";
 import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 import "org/apache/beam/model/pipeline/v1/schema.proto";
 
@@ -57,6 +58,10 @@ message ExpansionRequest {
   // A set of requirements that must be used by the expansion service to
   // interpret the components provided with this request.
   repeated string requirements = 5;
+
+  // (Optional) A set of Pipeline Options that should be used
+  // when expanding this transform.
+  google.protobuf.Struct pipeline_options = 6;
 }
 
 message ExpansionResponse {
diff --git a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto
index db958f183c45..08f05fc51b69 100644
--- a/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto
+++ b/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto
@@ -1563,13 +1563,26 @@ message Environment {
 
 message StandardEnvironments {
   enum Environments {
-    DOCKER = 0 [(beam_urn) = "beam:env:docker:v1"]; // A managed docker container to run user code.
-
-    PROCESS = 1 [(beam_urn) = "beam:env:process:v1"]; // A managed native process to run user code.
-
-    EXTERNAL = 2 [(beam_urn) = "beam:env:external:v1"]; // An external non managed process to run user code.
-
-    DEFAULT = 3 [(beam_urn) = "beam:env:default:v1"]; // Used as a stub when context is missing a runner-provided default environment.
+    // A managed docker container to run user code.
+    // Payload should be DockerPayload.
+    DOCKER = 0 [(beam_urn) = "beam:env:docker:v1"];
+
+    // A managed native process to run user code.
+    // Payload should be ProcessPayload.
+    PROCESS = 1 [(beam_urn) = "beam:env:process:v1"];
+
+    // An external non managed process to run user code.
+    // Payload should be ExternalPayload.
+    EXTERNAL = 2 [(beam_urn) = "beam:env:external:v1"];
+
+    // Used as a stub when context is missing a runner-provided default environment.
+    DEFAULT = 3 [(beam_urn) = "beam:env:default:v1"];
+
+    // A selection of equivalent fully-specified environments a runner may use.
+    // Note that this environment itself does not declare any dependencies or capabilities,
+    // as those may differ among the several alternatives.
+    // Payload should be AnyOfEnvironmentPayload.
+    ANYOF = 4 [(beam_urn) = "beam:env:anyof:v1"];
   }
 }
 
@@ -1586,10 +1599,15 @@ message ProcessPayload {
 }
 
 message ExternalPayload {
-  ApiServiceDescriptor endpoint = 1;
+  ApiServiceDescriptor endpoint = 1;  // Serving BeamFnExternalWorkerPool API.
   map<string, string> params = 2;  // Arbitrary extra parameters to pass
 }
 
+message AnyOfEnvironmentPayload {
+  // Each is fully contained (with their own dependencies, capabilities, etc.)
+  repeated Environment environments = 1;
+}
+
 // These URNs are used to indicate capabilities of environments that cannot
 // simply be expressed as a component (such as a Coder or PTransform) that this
 // environment understands.
@@ -1657,6 +1675,11 @@ message StandardRunnerProtocols {
     // https://s.apache.org/beam-fn-api-control-data-embedding
     CONTROL_RESPONSE_ELEMENTS_EMBEDDING = 6
     [(beam_urn) = "beam:protocol:control_response_elements_embedding:v1"];
+
+    // Indicates that this runner can handle the multimap_keys_values_side_input
+    // style read of a multimap side input.
+    MULTIMAP_KEYS_VALUES_SIDE_INPUT = 7
+    [(beam_urn) = "beam:protocol:multimap_keys_values_side_input:v1"];
   }
 }
 
diff --git a/release/go-licenses/Dockerfile b/release/go-licenses/Dockerfile
index 7008454ea6aa..1dfddc6e3745 100644
--- a/release/go-licenses/Dockerfile
+++ b/release/go-licenses/Dockerfile
@@ -16,7 +16,7 @@
 # limitations under the License.
 ###############################################################################
 
-FROM golang:1.20-bullseye
+FROM golang:1.20-bookworm
 
 WORKDIR /usr/src/app
 COPY go.mod ./
diff --git a/release/src/main/Dockerfile b/release/src/main/Dockerfile
index 04d39765cfc2..62d389889ea5 100644
--- a/release/src/main/Dockerfile
+++ b/release/src/main/Dockerfile
@@ -23,7 +23,7 @@
 #
 # then run `service docker start` and `docker login docker.io` to connect to the docker
 # daemon and authenticate with Docker Hub.
-FROM debian:bullseye
+FROM debian:bookworm
 LABEL Author "Apache Beam <dev@beam.apache.org>"
 SHELL ["/bin/bash", "-c"]
 
diff --git a/release/src/main/scripts/verify_release_build.sh b/release/src/main/scripts/verify_release_build.sh
index 51008d40831f..d304456ecb4a 100755
--- a/release/src/main/scripts/verify_release_build.sh
+++ b/release/src/main/scripts/verify_release_build.sh
@@ -27,9 +27,8 @@
 #      Instructions: https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line
 #   2. Please set RELEASE_BUILD_CONFIGS in script.config before running this
 #      script.
-#   3. Please manually comment trigger phrases to the created PR to start
-#      Gradle release build and all PostCommit jobs, or run mass_comment.py
-#      to do so. Phrases are listed in COMMENTS_TO_ADD in mass_comment.py.
+#   3. Please manually start "Release Nightly Snapshot" GHA workflow on the
+#      release branch to start Gradle release build.
 
 
 . script.config
@@ -144,9 +143,9 @@ if [[ ! -z `which hub` ]]; then
 
   hub pull-request -b apache:${RELEASE_BRANCH} -h ${GITHUB_USERNAME}:${WORKING_BRANCH} -F- <<<"[DO NOT MERGE] Run all PostCommit and PreCommit Tests against Release Branch
 
-  You can run many tests automatically using release/src/main/scripts/mass_comment.py."
+  Tests run on GHA should be queued shortly."
 
   echo ""
-  echo "[NOTE]: Please make sure all test targets (GHA and Jenkins) have been invoked."
+  echo "[NOTE]: Please make sure all test targets (GHA) have been invoked."
   echo "Please check the test results. If there is any failure, follow the policy in release guide."
 fi
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
index f531b5be344d..6e05c006d283 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/Environments.java
@@ -28,8 +28,11 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor;
 import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.AnyOfEnvironmentPayload;
 import org.apache.beam.model.pipeline.v1.RunnerApi.ArtifactInformation;
 import org.apache.beam.model.pipeline.v1.RunnerApi.Components;
 import org.apache.beam.model.pipeline.v1.RunnerApi.DockerPayload;
@@ -291,6 +294,50 @@ public static Environment createProcessEnvironment(
         .build();
   }
 
+  public static Environment createAnyOfEnvironment(Environment... environments) {
+    AnyOfEnvironmentPayload.Builder payload = AnyOfEnvironmentPayload.newBuilder();
+    for (Environment environment : environments) {
+      payload.addEnvironments(environment);
+    }
+    return Environment.newBuilder()
+        .setUrn(BeamUrns.getUrn(StandardEnvironments.Environments.ANYOF))
+        .setPayload(payload.build().toByteString())
+        .build();
+  }
+
+  public static List<Environment> expandAnyOfEnvironments(Environment environment) {
+    return Stream.of(environment)
+        .flatMap(
+            env -> {
+              if (BeamUrns.getUrn(StandardEnvironments.Environments.ANYOF)
+                  .equals(environment.getUrn())) {
+                try {
+                  return AnyOfEnvironmentPayload.parseFrom(environment.getPayload())
+                      .getEnvironmentsList().stream()
+                      .flatMap(subenv -> expandAnyOfEnvironments(subenv).stream());
+                } catch (InvalidProtocolBufferException exn) {
+                  throw new RuntimeException(exn);
+                }
+              } else {
+                return Stream.of(env);
+              }
+            })
+        .collect(Collectors.toList());
+  }
+
+  public static Environment resolveAnyOfEnvironment(
+      Environment environment, String... preferredEnvironmentTypes) {
+    List<Environment> allEnvironments = expandAnyOfEnvironments(environment);
+    for (String urn : preferredEnvironmentTypes) {
+      for (Environment env : allEnvironments) {
+        if (urn.equals(env.getUrn())) {
+          return env;
+        }
+      }
+    }
+    return allEnvironments.iterator().next();
+  }
+
   public static Optional<Environment> getEnvironment(String ptransformId, Components components) {
     PTransform ptransform = components.getTransformsOrThrow(ptransformId);
     String envId = ptransform.getEnvironmentId();
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java
index 93a1ade474a5..3951148d1a58 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/External.java
@@ -275,6 +275,7 @@ public OutputT expand(InputT input) {
               .setTransform(ptransformBuilder.build())
               .setNamespace(getNamespace())
               .build();
+      requestBuilder.setPipelineOptions(PipelineOptionsTranslation.toProto(p.getOptions()));
 
       ExpansionApi.ExpansionResponse response =
           clientFactory.getExpansionServiceClient(endpoint).expand(request);
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
index 8f415e718e95..6829e0d6b232 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PTransformTranslation.java
@@ -27,6 +27,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.ServiceLoader;
 import java.util.Set;
 import org.apache.beam.model.pipeline.v1.RunnerApi;
@@ -502,6 +503,12 @@ public RunnerApi.PTransform translate(
                 SchemaTranslation.schemaToProto(configRow.getSchema(), true).toByteArray()));
       }
 
+      for (Entry<String, byte[]> annotation :
+          appliedPTransform.getTransform().getAnnotations().entrySet()) {
+        transformBuilder.putAnnotations(
+            annotation.getKey(), ByteString.copyFrom(annotation.getValue()));
+      }
+
       return transformBuilder.build();
     }
   }
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
index 5ea2c4968dd9..9cf0606b68b9 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDo.java
@@ -21,6 +21,7 @@
 
 import com.google.auto.service.AutoService;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
@@ -669,6 +670,15 @@ public void output(RestrictionT part) {
                     public void outputWithTimestamp(RestrictionT part, Instant timestamp) {
                       throw new UnsupportedOperationException();
                     }
+
+                    @Override
+                    public void outputWindowedValue(
+                        RestrictionT output,
+                        Instant timestamp,
+                        Collection<? extends BoundedWindow> windows,
+                        PaneInfo paneInfo) {
+                      throw new UnsupportedOperationException();
+                    }
                   };
                 }
 
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java
index 061e1cb11b5c..ad6b51539742 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/SplittableParDoNaiveBounded.java
@@ -19,6 +19,7 @@
 
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import org.apache.beam.runners.core.construction.SplittableParDo.ProcessKeyedElements;
@@ -525,6 +526,15 @@ public void output(OutputT output) {
           public void outputWithTimestamp(OutputT output, Instant timestamp) {
             outerContext.outputWithTimestamp(output, timestamp);
           }
+
+          @Override
+          public void outputWindowedValue(
+              OutputT output,
+              Instant timestamp,
+              Collection<? extends BoundedWindow> windows,
+              PaneInfo paneInfo) {
+            outerContext.outputWindowedValue(output, timestamp, windows, paneInfo);
+          }
         };
       }
 
@@ -543,6 +553,15 @@ public void output(T output) {
               public void outputWithTimestamp(T output, Instant timestamp) {
                 outerContext.outputWithTimestamp(tag, output, timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  T output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                outerContext.outputWindowedValue(tag, output, timestamp, windows, paneInfo);
+              }
             };
           }
 
@@ -583,6 +602,15 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
         outerContext.outputWithTimestamp(output, timestamp);
       }
 
+      @Override
+      public void outputWindowedValue(
+          OutputT output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        outerContext.outputWindowedValue(output, timestamp, windows, paneInfo);
+      }
+
       @Override
       public <T> void output(TupleTag<T> tag, T output) {
         outerContext.output(tag, output);
@@ -593,6 +621,16 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
         outerContext.outputWithTimestamp(tag, output, timestamp);
       }
 
+      @Override
+      public <T> void outputWindowedValue(
+          TupleTag<T> tag,
+          T output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        outerContext.outputWindowedValue(tag, output, timestamp, windows, paneInfo);
+      }
+
       @Override
       public InputT element() {
         return element;
diff --git a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java
index 93fe4ddbfaf6..b142ab4af1c9 100644
--- a/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java
+++ b/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/TransformUpgrader.java
@@ -17,7 +17,12 @@
  */
 package org.apache.beam.runners.core.construction;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InvalidClassException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.net.ServerSocket;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -45,12 +50,16 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * A utility class that allows upgrading transforms of a given pipeline using the Beam Transform
  * Service.
  */
 public class TransformUpgrader implements AutoCloseable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TransformUpgrader.class);
   private static final String UPGRADE_NAMESPACE = "transform:upgrade:";
 
   private ExpansionServiceClientFactory clientFactory;
@@ -104,6 +113,15 @@ public RunnerApi.Pipeline upgradeTransformsViaTransformService(
                 })
             .collect(Collectors.toList());
 
+    if (!urnsToOverride.isEmpty() && transformsToOverride.isEmpty()) {
+      throw new IllegalArgumentException(
+          "A list of URNs for overriding transforms was provided but the pipeline did not contain "
+              + "any matching transforms. Either make sure to include at least one matching "
+              + "transform in the pipeline or avoid setting the 'transformsToOverride' "
+              + "PipelineOption. Provided list of URNs: "
+              + urnsToOverride);
+    }
+
     String serviceAddress;
     TransformServiceLauncher service = null;
 
@@ -232,11 +250,11 @@ RunnerApi.Pipeline updateTransformViaTransformService(
           expandedTransform.getOutputsMap().values().iterator().next());
     } else {
       for (Map.Entry<String, String> entry : transformToUpgrade.getOutputsMap().entrySet()) {
-        if (expandedTransform.getOutputsMap().keySet().contains(entry.getKey())) {
+        if (!expandedTransform.getOutputsMap().keySet().contains(entry.getKey())) {
           throw new IllegalArgumentException(
-              "Original transform did not have an output with tag "
+              "Original transform had an output with tag "
                   + entry.getKey()
-                  + " but upgraded transform did.");
+                  + " but upgraded transform did not.");
         }
         String newOutput = expandedTransform.getOutputsMap().get(entry.getKey());
         if (newOutput == null) {
@@ -367,4 +385,43 @@ public void close() throws Exception {
 
     return null;
   }
+
+  /**
+   * A utility method that converts an arbitrary serializable object into a byte array.
+   *
+   * @param object an instance of type {@code Serializable}
+   * @return {@code object} converted into a byte array.
+   */
+  public static byte[] toByteArray(Object object) {
+    try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        ObjectOutputStream out = new ObjectOutputStream(bos)) {
+      out.writeObject(object);
+      return bos.toByteArray();
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * A utility method that converts a byte array obtained by invoking {@link #toByteArray(Object)}
+   * back to a Java object.
+   *
+   * @param bytes a {@code byte} array generated by invoking the {@link #toByteArray(Object)}
+   *     method.
+   * @return re-generated object.
+   */
+  public static Object fromByteArray(byte[] bytes) throws InvalidClassException {
+    try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
+        ObjectInputStream in = new ObjectInputStream(bis)) {
+      return in.readObject();
+    } catch (InvalidClassException e) {
+      LOG.info(
+          "An object cannot be re-generated from the provided byte array. Caller may use the "
+              + "default value for the parameter when upgrading. Underlying error: "
+              + e);
+      throw e;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
 }
diff --git a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java
index b71a654f1031..453f6ab6db88 100644
--- a/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java
+++ b/runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/EnvironmentsTest.java
@@ -19,15 +19,18 @@
 
 import static org.apache.beam.runners.core.construction.Environments.JAVA_SDK_HARNESS_CONTAINER_URL;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
 import static org.junit.Assert.assertEquals;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 import org.apache.beam.model.pipeline.v1.Endpoints;
@@ -353,4 +356,42 @@ public void testGetArtifactsBadNamedFileLogsWarn() throws Exception {
     assertThat(artifacts, hasSize(1));
     expectedLogs.verifyWarn("name 'file_name' was not found");
   }
+
+  @Test
+  public void testExpandAnyOfEnvironmentsOnOrdinaryEnvironment() {
+    Environment env = Environments.createDockerEnvironment("java");
+    assertThat(Environments.expandAnyOfEnvironments(env), contains(env));
+  }
+
+  @Test
+  public void testExpandAnyOfEnvironmentsOnNestedEnvironment() {
+    Environment envA = Environments.createDockerEnvironment("A");
+    Environment envB = Environments.createDockerEnvironment("B");
+    Environment envC = Environments.createDockerEnvironment("C");
+    Environment env =
+        Environments.createAnyOfEnvironment(envA, Environments.createAnyOfEnvironment(envB, envC));
+    assertThat(Environments.expandAnyOfEnvironments(env), contains(envA, envB, envC));
+  }
+
+  @Test
+  public void testResolveAnyOfEnvironment() {
+    Environment dockerEnv = Environments.createDockerEnvironment("A");
+    Environment processEnv =
+        Environments.createProcessEnvironment("os", "arch", "cmd", new HashMap<>());
+    Environment env =
+        Environments.createAnyOfEnvironment(
+            dockerEnv, Environments.createAnyOfEnvironment(processEnv));
+    assertThat(
+        Environments.resolveAnyOfEnvironment(
+            env, BeamUrns.getUrn(StandardEnvironments.Environments.DOCKER)),
+        equalTo(dockerEnv));
+    assertThat(
+        Environments.resolveAnyOfEnvironment(
+            env, BeamUrns.getUrn(StandardEnvironments.Environments.PROCESS)),
+        equalTo(processEnv));
+    assertThat(
+        Environments.resolveAnyOfEnvironment(
+            env, BeamUrns.getUrn(StandardEnvironments.Environments.EXTERNAL)),
+        notNullValue());
+  }
 }
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
index 85a46eb7dc04..51cd8c690aee 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/OutputAndTimeBoundedSplittableProcessElementInvoker.java
@@ -20,6 +20,7 @@
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState;
 
+import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.CancellationException;
 import java.util.concurrent.Future;
@@ -42,6 +43,7 @@
 import org.apache.beam.sdk.transforms.splittabledofn.SplitResult;
 import org.apache.beam.sdk.transforms.splittabledofn.TimestampObservingWatermarkEstimator;
 import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
@@ -388,11 +390,20 @@ public void output(OutputT output) {
 
     @Override
     public void outputWithTimestamp(OutputT value, Instant timestamp) {
+      outputWindowedValue(value, timestamp, element.getWindows(), element.getPane());
+    }
+
+    @Override
+    public void outputWindowedValue(
+        OutputT value,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
       noteOutput();
       if (watermarkEstimator instanceof TimestampObservingWatermarkEstimator) {
         ((TimestampObservingWatermarkEstimator) watermarkEstimator).observeTimestamp(timestamp);
       }
-      output.outputWindowedValue(value, timestamp, element.getWindows(), element.getPane());
+      output.outputWindowedValue(value, timestamp, windows, paneInfo);
     }
 
     @Override
@@ -402,11 +413,21 @@ public <T> void output(TupleTag<T> tag, T value) {
 
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T value, Instant timestamp) {
+      outputWindowedValue(tag, value, timestamp, element.getWindows(), element.getPane());
+    }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T value,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
       noteOutput();
       if (watermarkEstimator instanceof TimestampObservingWatermarkEstimator) {
         ((TimestampObservingWatermarkEstimator) watermarkEstimator).observeTimestamp(timestamp);
       }
-      output.outputWindowedValue(tag, value, timestamp, element.getWindows(), element.getPane());
+      output.outputWindowedValue(tag, value, timestamp, windows, paneInfo);
     }
 
     private void noteOutput() {
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
index 16986fdf8d52..b375d38c5a98 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/SimpleDoFnRunner.java
@@ -21,6 +21,7 @@
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -413,22 +414,40 @@ public void output(OutputT output) {
 
     @Override
     public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      checkTimestamp(elem.getTimestamp(), timestamp);
       outputWithTimestamp(mainOutputTag, output, timestamp);
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      outputWindowedValue(mainOutputTag, output, timestamp, windows, paneInfo);
+    }
+
     @Override
     public <T> void output(TupleTag<T> tag, T output) {
       checkNotNull(tag, "Tag passed to output cannot be null");
-      outputWindowedValue(tag, elem.withValue(output));
+      SimpleDoFnRunner.this.outputWindowedValue(tag, elem.withValue(output));
     }
 
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       checkNotNull(tag, "Tag passed to outputWithTimestamp cannot be null");
       checkTimestamp(elem.getTimestamp(), timestamp);
-      outputWindowedValue(
-          tag, WindowedValue.of(output, timestamp, elem.getWindows(), elem.getPane()));
+      outputWindowedValue(tag, output, timestamp, elem.getWindows(), elem.getPane());
+    }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      SimpleDoFnRunner.this.outputWindowedValue(
+          tag, WindowedValue.of(output, timestamp, windows, paneInfo));
     }
 
     @Override
@@ -838,16 +857,38 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
       outputWithTimestamp(mainOutputTag, output, timestamp);
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      outputWindowedValue(mainOutputTag, output, timestamp, windows, paneInfo);
+    }
+
     @Override
     public <T> void output(TupleTag<T> tag, T output) {
       checkTimestamp(timestamp(), timestamp);
-      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
+      outputWithTimestamp(tag, output, timestamp);
     }
 
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       checkTimestamp(timestamp(), timestamp);
-      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
+      outputWindowedValue(
+          tag, output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+    }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      checkTimestamp(timestamp(), timestamp);
+      SimpleDoFnRunner.this.outputWindowedValue(
+          tag, WindowedValue.of(output, timestamp, windows, paneInfo));
     }
 
     @Override
@@ -1045,16 +1086,38 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
       outputWithTimestamp(mainOutputTag, output, timestamp);
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      outputWindowedValue(mainOutputTag, output, timestamp, windows, paneInfo);
+    }
+
     @Override
     public <T> void output(TupleTag<T> tag, T output) {
       checkTimestamp(this.timestamp, timestamp);
-      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
+      outputWithTimestamp(tag, output, timestamp);
     }
 
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       checkTimestamp(this.timestamp, timestamp);
-      outputWindowedValue(tag, WindowedValue.of(output, timestamp, window(), PaneInfo.NO_FIRING));
+      outputWindowedValue(
+          tag, output, timestamp, Collections.singleton(window()), PaneInfo.NO_FIRING);
+    }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      checkTimestamp(this.timestamp, timestamp);
+      SimpleDoFnRunner.this.outputWindowedValue(
+          tag, WindowedValue.of(output, timestamp, windows, paneInfo));
     }
 
     @Override
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java
index a2b745c74ede..9478c218b075 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateSampler.java
@@ -44,10 +44,10 @@ public class ExecutionStateSampler {
   private static final ExecutionStateSampler INSTANCE =
       new ExecutionStateSampler(SYSTEM_MILLIS_PROVIDER);
 
-  private final MillisProvider clock;
-  @VisibleForTesting volatile long lastSampleTimeMillis;
+  protected final MillisProvider clock;
+  @VisibleForTesting protected volatile long lastSampleTimeMillis;
 
-  private ExecutionStateSampler(MillisProvider clock) {
+  protected ExecutionStateSampler(MillisProvider clock) {
     this.clock = clock;
   }
 
@@ -147,12 +147,12 @@ public synchronized void stop() {
   }
 
   /** Add the tracker to the sampling set. */
-  void addTracker(ExecutionStateTracker tracker) {
+  protected void addTracker(ExecutionStateTracker tracker) {
     this.activeTrackers.add(tracker);
   }
 
   /** Remove the tracker from the sampling set. */
-  void removeTracker(ExecutionStateTracker tracker) {
+  protected void removeTracker(ExecutionStateTracker tracker) {
     activeTrackers.remove(tracker);
 
     // Attribute any remaining time since the last sampling while removing the tracker.
diff --git a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java
index 0eae4a93245c..dc6fd2f8248c 100644
--- a/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java
+++ b/runners/core-java/src/main/java/org/apache/beam/runners/core/metrics/ExecutionStateTracker.java
@@ -304,7 +304,14 @@ public long getNextLullReportMs() {
     return nextLullReportMs;
   }
 
-  void takeSample(long millisSinceLastSample) {
+  /**
+   * Called periodically by the {@link ExecutionStateSampler} to report time recorded by the
+   * tracker.
+   *
+   * @param millisSinceLastSample the time since the last sample was reported. As an approximation,
+   *     all of that time should be associated with this tracker.
+   */
+  public void takeSample(long millisSinceLastSample) {
     if (SAMPLING_UPDATER.compareAndSet(this, 0, 1)) {
       try {
         takeSampleOnce(millisSinceLastSample);
diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle
index c510b346d5d0..c5946c509a95 100644
--- a/runners/flink/flink_runner.gradle
+++ b/runners/flink/flink_runner.gradle
@@ -310,6 +310,8 @@ def createValidatesRunnerTask(Map m) {
           excludeTestsMatching test
         }
 
+        // Flink reshuffle override does not preserve all metadata
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
         // https://github.com/apache/beam/issues/20843
         excludeTestsMatching 'org.apache.beam.sdk.testing.TestStreamTest.testDiscardingMode'
         // https://github.com/apache/beam/issues/20845
diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle
index 9643134d95b2..ab3f726e6ad9 100644
--- a/runners/flink/job-server/flink_job_server.gradle
+++ b/runners/flink/job-server/flink_job_server.gradle
@@ -196,6 +196,8 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean checkpoi
         excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderInBundle'
     },
     testFilter: {
+      // Flink reshuffle override does not preserve all metadata
+      excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
       // TODO(https://github.com/apache/beam/issues/20269)
       excludeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2'
       // TODO(https://github.com/apache/beam/issues/20843)
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderBase.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderBase.java
index f0b93e0dde0f..ce4404f8ce9a 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderBase.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderBase.java
@@ -17,8 +17,6 @@
  */
 package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -42,7 +40,6 @@
 import org.apache.beam.runners.flink.metrics.FlinkMetricContainerWithoutAccumulator;
 import org.apache.beam.runners.flink.metrics.ReaderInvocationUtil;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.compat.FlinkSourceCompat;
-import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
@@ -149,19 +146,11 @@ public List<FlinkSourceSplit<T>> snapshotState(long checkpointId) {
     // Add all the source splits being actively read.
     beamSourceReaders.forEach(
         (splitId, readerAndOutput) -> {
-          Source.Reader<T> reader = readerAndOutput.reader;
-          if (reader instanceof BoundedSource.BoundedReader) {
-            // Sometimes users may decide to run a bounded source in streaming mode as "finite
-            // stream."
-            // For bounded source, the checkpoint granularity is the entire source split.
-            // So, in case of failure, all the data from this split will be consumed again.
-            splitsState.add(new FlinkSourceSplit<>(splitId, reader.getCurrentSource()));
-          } else if (reader instanceof UnboundedSource.UnboundedReader) {
-            // The checkpoint for unbounded sources is fine granular.
-            byte[] checkpointState =
-                getAndEncodeCheckpointMark((UnboundedSource.UnboundedReader<OutputT>) reader);
-            splitsState.add(
-                new FlinkSourceSplit<>(splitId, reader.getCurrentSource(), checkpointState));
+          try {
+            splitsState.add(getReaderCheckpoint(splitId, readerAndOutput));
+          } catch (IOException e) {
+            throw new IllegalStateException(
+                String.format("Failed to get checkpoint for split %d", splitId), e);
           }
         });
     return splitsState;
@@ -228,9 +217,17 @@ public void close() throws Exception {
    */
   protected abstract CompletableFuture<Void> isAvailableForAliveReaders();
 
+  /** Create {@link FlinkSourceSplit} for given {@code splitId}. */
+  protected abstract FlinkSourceSplit<T> getReaderCheckpoint(
+      int splitId, ReaderAndOutput readerAndOutput) throws IOException;
+
+  /** Create {@link Source.Reader} for given {@link FlinkSourceSplit}. */
+  protected abstract Source.Reader<T> createReader(@Nonnull FlinkSourceSplit<T> sourceSplit)
+      throws IOException;
+
   // ----------------- protected helper methods for subclasses --------------------
 
-  protected Optional<ReaderAndOutput> createAndTrackNextReader() throws IOException {
+  protected final Optional<ReaderAndOutput> createAndTrackNextReader() throws IOException {
     FlinkSourceSplit<T> sourceSplit = sourceSplits.poll();
     if (sourceSplit != null) {
       Source.Reader<T> reader = createReader(sourceSplit);
@@ -241,7 +238,7 @@ protected Optional<ReaderAndOutput> createAndTrackNextReader() throws IOExceptio
     return Optional.empty();
   }
 
-  protected void finishSplit(int splitIndex) throws IOException {
+  protected final void finishSplit(int splitIndex) throws IOException {
     ReaderAndOutput readerAndOutput = beamSourceReaders.remove(splitIndex);
     if (readerAndOutput != null) {
       LOG.info("Finished reading from split {}", readerAndOutput.splitId);
@@ -252,7 +249,7 @@ protected void finishSplit(int splitIndex) throws IOException {
     }
   }
 
-  protected boolean checkIdleTimeoutAndMaybeStartCountdown() {
+  protected final boolean checkIdleTimeoutAndMaybeStartCountdown() {
     if (idleTimeoutMs <= 0) {
       idleTimeoutFuture.complete(null);
     } else if (!idleTimeoutCountingDown) {
@@ -262,7 +259,7 @@ protected boolean checkIdleTimeoutAndMaybeStartCountdown() {
     return idleTimeoutFuture.isDone();
   }
 
-  protected boolean noMoreSplits() {
+  protected final boolean noMoreSplits() {
     return noMoreSplits;
   }
 
@@ -308,49 +305,6 @@ protected Map<Integer, ReaderAndOutput> allReaders() {
   protected static void ignoreReturnValue(Object o) {
     // do nothing.
   }
-  // ------------------------------ private methods ------------------------------
-
-  @SuppressWarnings("unchecked")
-  private <CheckpointMarkT extends UnboundedSource.CheckpointMark>
-      byte[] getAndEncodeCheckpointMark(UnboundedSource.UnboundedReader<OutputT> reader) {
-    UnboundedSource<OutputT, CheckpointMarkT> source =
-        (UnboundedSource<OutputT, CheckpointMarkT>) reader.getCurrentSource();
-    CheckpointMarkT checkpointMark = (CheckpointMarkT) reader.getCheckpointMark();
-    Coder<CheckpointMarkT> coder = source.getCheckpointMarkCoder();
-    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
-      coder.encode(checkpointMark, baos);
-      return baos.toByteArray();
-    } catch (IOException ioe) {
-      throw new RuntimeException("Failed to encode checkpoint mark.", ioe);
-    }
-  }
-
-  private Source.Reader<T> createReader(@Nonnull FlinkSourceSplit<T> sourceSplit)
-      throws IOException {
-    Source<T> beamSource = sourceSplit.getBeamSplitSource();
-    if (beamSource instanceof BoundedSource) {
-      return ((BoundedSource<T>) beamSource).createReader(pipelineOptions);
-    } else if (beamSource instanceof UnboundedSource) {
-      return createUnboundedSourceReader(beamSource, sourceSplit.getSplitState());
-    } else {
-      throw new IllegalStateException("Unknown source type " + beamSource.getClass());
-    }
-  }
-
-  private <CheckpointMarkT extends UnboundedSource.CheckpointMark>
-      Source.Reader<T> createUnboundedSourceReader(
-          Source<T> beamSource, @Nullable byte[] splitState) throws IOException {
-    UnboundedSource<T, CheckpointMarkT> unboundedSource =
-        (UnboundedSource<T, CheckpointMarkT>) beamSource;
-    Coder<CheckpointMarkT> coder = unboundedSource.getCheckpointMarkCoder();
-    if (splitState == null) {
-      return unboundedSource.createReader(pipelineOptions, null);
-    } else {
-      try (ByteArrayInputStream bais = new ByteArrayInputStream(splitState)) {
-        return unboundedSource.createReader(pipelineOptions, coder.decode(bais));
-      }
-    }
-  }
 
   // -------------------- protected helper class ---------------------
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java
index 292697479bcd..8ceab393533d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java
@@ -121,7 +121,6 @@ public void addReader(int subtaskId) {
     List<FlinkSourceSplit<T>> splitsForSubtask = pendingSplits.remove(subtaskId);
     if (splitsForSubtask != null) {
       assignSplitsAndLog(splitsForSubtask, subtaskId);
-      pendingSplits.remove(subtaskId);
     } else {
       if (splitsInitialized) {
         LOG.info("There is no split for subtask {}. Signaling no more splits.", subtaskId);
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java
index b015b527aa45..a25964af809d 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReader.java
@@ -18,18 +18,29 @@
 package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.bounded;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.function.Function;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceReaderBase;
+import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceSplit;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.io.BoundedSource;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.PaneInfo;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.Preconditions;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects;
+import org.apache.flink.api.common.eventtime.Watermark;
 import org.apache.flink.api.connector.source.ReaderOutput;
 import org.apache.flink.api.connector.source.SourceReaderContext;
 import org.apache.flink.core.io.InputStatus;
@@ -50,6 +61,8 @@
  */
 public class FlinkBoundedSourceReader<T> extends FlinkSourceReaderBase<T, WindowedValue<T>> {
   private static final Logger LOG = LoggerFactory.getLogger(FlinkBoundedSourceReader.class);
+  private static final VarLongCoder LONG_CODER = VarLongCoder.of();
+  private final Map<Integer, Long> consumedFromSplit = new HashMap<>();
   private @Nullable Source.Reader<T> currentReader;
   private int currentSplitId;
 
@@ -62,6 +75,40 @@ public FlinkBoundedSourceReader(
     currentSplitId = -1;
   }
 
+  @Override
+  protected FlinkSourceSplit<T> getReaderCheckpoint(int splitId, ReaderAndOutput readerAndOutput)
+      throws CoderException {
+    // Sometimes users may decide to run a bounded source in streaming mode as "finite
+    // stream."
+    // For bounded source, the checkpoint granularity is the entire source split.
+    // So, in case of failure, all the data from this split will be consumed again.
+    return new FlinkSourceSplit<>(
+        splitId, readerAndOutput.reader.getCurrentSource(), asBytes(consumedFromSplit(splitId)));
+  }
+
+  @Override
+  protected Source.Reader<T> createReader(@Nonnull FlinkSourceSplit<T> sourceSplit)
+      throws IOException {
+    Source<T> beamSource = sourceSplit.getBeamSplitSource();
+    byte[] state = sourceSplit.getSplitState();
+    if (state != null) {
+      consumedFromSplit.put(Integer.parseInt(sourceSplit.splitId()), fromBytes(state));
+    }
+    return ((BoundedSource<T>) beamSource).createReader(pipelineOptions);
+  }
+
+  private byte[] asBytes(long l) throws CoderException {
+    return CoderUtils.encodeToByteArray(LONG_CODER, l);
+  }
+
+  private long fromBytes(byte[] b) throws CoderException {
+    return CoderUtils.decodeFromByteArray(LONG_CODER, b);
+  }
+
+  private long consumedFromSplit(int splitId) {
+    return consumedFromSplit.getOrDefault(splitId, 0L);
+  }
+
   @VisibleForTesting
   protected FlinkBoundedSourceReader(
       String stepName,
@@ -78,26 +125,28 @@ public InputStatus pollNext(ReaderOutput<WindowedValue<T>> output) throws Except
     checkExceptionAndMaybeThrow();
     if (currentReader == null && !moveToNextNonEmptyReader()) {
       // Nothing to read for now.
-      if (noMoreSplits() && checkIdleTimeoutAndMaybeStartCountdown()) {
-        // All the source splits have been read and idle timeout has passed.
-        LOG.info(
-            "All splits have finished reading, and idle time {} ms has passed.", idleTimeoutMs);
-        return InputStatus.END_OF_INPUT;
-      } else {
-        // This reader either hasn't received NoMoreSplitsEvent yet or it is waiting for idle
-        // timeout.
-        return InputStatus.NOTHING_AVAILABLE;
+      if (noMoreSplits()) {
+        output.emitWatermark(Watermark.MAX_WATERMARK);
+        if (checkIdleTimeoutAndMaybeStartCountdown()) {
+          // All the source splits have been read and idle timeout has passed.
+          LOG.info(
+              "All splits have finished reading, and idle time {} ms has passed.", idleTimeoutMs);
+          return InputStatus.END_OF_INPUT;
+        }
       }
+      // This reader either hasn't received NoMoreSplitsEvent yet or it is waiting for idle
+      // timeout.
+      return InputStatus.NOTHING_AVAILABLE;
     }
-    Source.Reader<T> tempCurrentReader = currentReader;
-    if (tempCurrentReader != null) {
-      T record = tempCurrentReader.getCurrent();
+    if (currentReader != null) {
+      // make null checks happy
+      final @Nonnull Source.Reader<T> splitReader = currentReader;
+      // store number of processed elements from this split
+      consumedFromSplit.compute(currentSplitId, (k, v) -> v == null ? 1 : v + 1);
+      T record = splitReader.getCurrent();
       WindowedValue<T> windowedValue =
           WindowedValue.of(
-              record,
-              tempCurrentReader.getCurrentTimestamp(),
-              GlobalWindow.INSTANCE,
-              PaneInfo.NO_FIRING);
+              record, splitReader.getCurrentTimestamp(), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING);
       if (timestampExtractor == null) {
         output.collect(windowedValue);
       } else {
@@ -107,11 +156,12 @@ public InputStatus pollNext(ReaderOutput<WindowedValue<T>> output) throws Except
       // If the advance() invocation throws exception here, the job will just fail over and read
       // everything again from
       // the beginning. So the failover granularity is the entire Flink job.
-      if (!invocationUtil.invokeAdvance(tempCurrentReader)) {
+      if (!invocationUtil.invokeAdvance(splitReader)) {
         finishSplit(currentSplitId);
+        consumedFromSplit.remove(currentSplitId);
+        LOG.debug("Finished reading from {}", currentSplitId);
         currentReader = null;
         currentSplitId = -1;
-        LOG.debug("Finished reading from {}", currentSplitId);
       }
       // Always return MORE_AVAILABLE here regardless of the availability of next record. If there
       // is no more
@@ -138,6 +188,12 @@ private boolean moveToNextNonEmptyReader() throws IOException {
       if (invocationUtil.invokeStart(rao.reader)) {
         currentSplitId = Integer.parseInt(rao.splitId);
         currentReader = rao.reader;
+        long toSkipAfterStart =
+            MoreObjects.firstNonNull(consumedFromSplit.remove(currentSplitId), 0L);
+        @Nonnull Source.Reader<T> reader = Preconditions.checkArgumentNotNull(currentReader);
+        while (toSkipAfterStart > 0 && reader.advance()) {
+          toSkipAfterStart--;
+        }
         return true;
       } else {
         finishSplit(Integer.parseInt(rao.splitId));
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java
index 0a7acb669efd..7b02702e244c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/unbounded/FlinkUnboundedSourceReader.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.unbounded;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -25,9 +27,12 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.apache.beam.runners.flink.FlinkPipelineOptions;
 import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceReaderBase;
+import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceSplit;
+import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.io.UnboundedSource;
 import org.apache.beam.sdk.options.PipelineOptions;
@@ -179,6 +184,22 @@ protected CompletableFuture<Void> isAvailableForAliveReaders() {
     }
   }
 
+  @Override
+  protected FlinkSourceSplit<T> getReaderCheckpoint(int splitId, ReaderAndOutput readerAndOutput) {
+    // The checkpoint for unbounded sources is fine granular.
+    byte[] checkpointState =
+        getAndEncodeCheckpointMark((UnboundedSource.UnboundedReader<T>) readerAndOutput.reader);
+    return new FlinkSourceSplit<>(
+        splitId, readerAndOutput.reader.getCurrentSource(), checkpointState);
+  }
+
+  @Override
+  protected Source.Reader<T> createReader(@Nonnull FlinkSourceSplit<T> sourceSplit)
+      throws IOException {
+    Source<T> beamSource = sourceSplit.getBeamSplitSource();
+    return createUnboundedSourceReader(beamSource, sourceSplit.getSplitState());
+  }
+
   // -------------- private helper methods ----------------
 
   private void emitRecord(
@@ -274,4 +295,34 @@ private void createPendingBytesGauge(SourceReaderContext context) {
               return pendingBytes;
             });
   }
+
+  @SuppressWarnings("unchecked")
+  private <CheckpointMarkT extends UnboundedSource.CheckpointMark>
+      byte[] getAndEncodeCheckpointMark(UnboundedSource.UnboundedReader<T> reader) {
+    UnboundedSource<T, CheckpointMarkT> source =
+        (UnboundedSource<T, CheckpointMarkT>) reader.getCurrentSource();
+    CheckpointMarkT checkpointMark = (CheckpointMarkT) reader.getCheckpointMark();
+    Coder<CheckpointMarkT> coder = source.getCheckpointMarkCoder();
+    try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+      coder.encode(checkpointMark, baos);
+      return baos.toByteArray();
+    } catch (IOException ioe) {
+      throw new RuntimeException("Failed to encode checkpoint mark.", ioe);
+    }
+  }
+
+  private <CheckpointMarkT extends UnboundedSource.CheckpointMark>
+      Source.Reader<T> createUnboundedSourceReader(
+          Source<T> beamSource, @Nullable byte[] splitState) throws IOException {
+    UnboundedSource<T, CheckpointMarkT> unboundedSource =
+        (UnboundedSource<T, CheckpointMarkT>) beamSource;
+    Coder<CheckpointMarkT> coder = unboundedSource.getCheckpointMarkCoder();
+    if (splitState == null) {
+      return unboundedSource.createReader(pipelineOptions, null);
+    } else {
+      try (ByteArrayInputStream bais = new ByteArrayInputStream(splitState)) {
+        return unboundedSource.createReader(pipelineOptions, coder.decode(bais));
+      }
+    }
+  }
 }
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderTestBase.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderTestBase.java
index 462a1ba0153d..c635a5778b5c 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderTestBase.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceReaderTestBase.java
@@ -363,11 +363,9 @@ public int numCollectedRecords() {
     }
 
     public boolean allRecordsConsumed() {
-      boolean allRecordsConsumed = true;
-      for (Source<?> source : sources) {
-        allRecordsConsumed = allRecordsConsumed && ((TestSource) source).isConsumptionCompleted();
-      }
-      return allRecordsConsumed;
+      return sources.stream()
+          .map(TestSource.class::cast)
+          .allMatch(TestSource::isConsumptionCompleted);
     }
 
     public boolean allTimestampReceived() {
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReaderTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReaderTest.java
index 84cb2a72ddaf..022f1abde826 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReaderTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/bounded/FlinkBoundedSourceReaderTest.java
@@ -20,6 +20,7 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.verify;
 
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
@@ -34,6 +35,7 @@
 import org.apache.beam.sdk.io.Source;
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
+import org.apache.flink.api.common.eventtime.Watermark;
 import org.apache.flink.api.connector.source.ReaderOutput;
 import org.apache.flink.api.connector.source.SourceReader;
 import org.apache.flink.api.connector.source.SourceReaderContext;
@@ -62,6 +64,20 @@ public void testPollWithIdleTimeout() throws Exception {
     }
   }
 
+  @Test
+  public void testPollEmitsMaxWatermark() throws Exception {
+    ManuallyTriggeredScheduledExecutorService executor =
+        new ManuallyTriggeredScheduledExecutorService();
+    ReaderOutput<WindowedValue<KV<Integer, Integer>>> mockReaderOutput =
+        Mockito.mock(ReaderOutput.class);
+    try (FlinkBoundedSourceReader<KV<Integer, Integer>> reader =
+        (FlinkBoundedSourceReader<KV<Integer, Integer>>) createReader(executor, Long.MAX_VALUE)) {
+      reader.notifyNoMoreSplits();
+      assertEquals(InputStatus.NOTHING_AVAILABLE, reader.pollNext(mockReaderOutput));
+      verify(mockReaderOutput).emitWatermark(Watermark.MAX_WATERMARK);
+    }
+  }
+
   @Test
   public void testPollWithoutIdleTimeout() throws Exception {
     ReaderOutput<WindowedValue<KV<Integer, Integer>>> mockReaderOutput =
@@ -107,8 +123,6 @@ public void testSnapshotStateAndRestore() throws Exception {
       snapshot = reader.snapshotState(0L);
     }
 
-    // Create a new validating output because the first split will be consumed from very beginning.
-    validatingOutput = new RecordsValidatingOutput(splits);
     // Create another reader, add the snapshot splits back.
     try (SourceReader<WindowedValue<KV<Integer, Integer>>, FlinkSourceSplit<KV<Integer, Integer>>>
         reader = createReader()) {
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 5d967d90f2b8..c7e5e1ba6236 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -51,7 +51,7 @@ evaluationDependsOn(":sdks:java:container:java11")
 
 ext.dataflowLegacyEnvironmentMajorVersion = '8'
 ext.dataflowFnapiEnvironmentMajorVersion = '8'
-ext.dataflowLegacyContainerVersion = 'beam-master-20231026'
+ext.dataflowLegacyContainerVersion = 'beam-master-20231212'
 ext.dataflowFnapiContainerVersion = 'beam-master-20231026'
 ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3'
 
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
index 0d9328fe136d..5f8098768261 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -877,6 +877,21 @@ public PTransformReplacement<PCollection<InputT>, PCollection<OutputT>> getRepla
     }
   }
 
+  private RunnerApi.Pipeline resolveAnyOfEnvironments(RunnerApi.Pipeline pipeline) {
+    RunnerApi.Pipeline.Builder pipelineBuilder = pipeline.toBuilder();
+    RunnerApi.Components.Builder componentsBuilder = pipelineBuilder.getComponentsBuilder();
+    componentsBuilder.clearEnvironments();
+    for (Map.Entry<String, RunnerApi.Environment> entry :
+        pipeline.getComponents().getEnvironmentsMap().entrySet()) {
+      componentsBuilder.putEnvironments(
+          entry.getKey(),
+          Environments.resolveAnyOfEnvironment(
+              entry.getValue(),
+              BeamUrns.getUrn(RunnerApi.StandardEnvironments.Environments.DOCKER)));
+    }
+    return pipelineBuilder.build();
+  }
+
   protected RunnerApi.Pipeline applySdkEnvironmentOverrides(
       RunnerApi.Pipeline pipeline, DataflowPipelineOptions options) {
     String sdkHarnessContainerImageOverrides = options.getSdkHarnessContainerImageOverrides();
@@ -1173,6 +1188,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
         PipelineTranslation.toProto(pipeline, portableComponents, false);
     // Note that `stageArtifacts` has to be called before `resolveArtifact` because
     // `resolveArtifact` updates local paths to staged paths in pipeline proto.
+    portablePipelineProto = resolveAnyOfEnvironments(portablePipelineProto);
     List<DataflowPackage> packages = stageArtifacts(portablePipelineProto);
     portablePipelineProto = resolveArtifacts(portablePipelineProto);
     portablePipelineProto = applySdkEnvironmentOverrides(portablePipelineProto, options);
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
index efb725f6a67b..b00194dacb08 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/TestDataflowRunner.java
@@ -23,6 +23,7 @@
 import com.google.api.services.dataflow.model.JobMetrics;
 import com.google.api.services.dataflow.model.MetricUpdate;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import java.io.File;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.util.List;
@@ -33,12 +34,12 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult.State;
 import org.apache.beam.sdk.PipelineRunner;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Strings;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -74,8 +75,15 @@ public class TestDataflowRunner extends PipelineRunner<DataflowPipelineJob> {
   public static TestDataflowRunner fromOptions(PipelineOptions options) {
     TestDataflowPipelineOptions dataflowOptions = options.as(TestDataflowPipelineOptions.class);
     String tempLocation =
-        Joiner.on("/")
-            .join(dataflowOptions.getTempRoot(), dataflowOptions.getJobName(), "output", "results");
+        FileSystems.matchNewDirectory(
+                dataflowOptions.getTempRoot(), dataflowOptions.getJobName(), "output", "results")
+            .toString();
+    // to keep exact same behavior prior to matchNewDirectory introduced
+    if (tempLocation.endsWith("/")) {
+      tempLocation = tempLocation.substring(0, tempLocation.length() - 1);
+    } else if (tempLocation.endsWith(File.separator)) {
+      tempLocation = tempLocation.substring(0, tempLocation.length() - File.separator.length());
+    }
     dataflowOptions.setTempLocation(tempLocation);
 
     return new TestDataflowRunner(
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
index e68dde045763..fcfe3fe3ce05 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/CustomSources.java
@@ -49,6 +49,10 @@ public class CustomSources {
   private static final Logger LOG = LoggerFactory.getLogger(CustomSources.class);
 
   private static int getDesiredNumUnboundedSourceSplits(DataflowPipelineOptions options) {
+    if (options.getDesiredNumUnboundedSourceSplits() > 0) {
+      return options.getDesiredNumUnboundedSourceSplits();
+    }
+
     int cores = 4; // TODO: decide at runtime?
     if (options.getMaxNumWorkers() > 0) {
       return options.getMaxNumWorkers() * cores;
diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
index 709009f8c806..f8649a1f0f3e 100644
--- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
+++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java
@@ -280,6 +280,16 @@ public Dataflow create(PipelineOptions options) {
 
   void setUnboundedReaderMaxWaitForElementsMs(Integer value);
 
+  /**
+   * The desired number of initial splits for UnboundedSources. If this value is <=0, the splits
+   * will be computed based on the number of user workers.
+   */
+  @Description("The desired number of initial splits for UnboundedSources.")
+  @Default.Integer(0)
+  int getDesiredNumUnboundedSourceSplits();
+
+  void setDesiredNumUnboundedSourceSplits(int value);
+
   /**
    * CAUTION: This option implies dumpHeapOnOOM, and has similar caveats. Specifically, heap dumps
    * can of comparable size to the default boot disk. Consider increasing the boot disk size before
diff --git a/.test-infra/jenkins/job_PostCommit_Website_Test.groovy b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ActiveMessageMetadata.java
similarity index 56%
rename from .test-infra/jenkins/job_PostCommit_Website_Test.groovy
rename to runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ActiveMessageMetadata.java
index 6c1c9cc48f5b..bc6b930a432d 100644
--- a/.test-infra/jenkins/job_PostCommit_Website_Test.groovy
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ActiveMessageMetadata.java
@@ -15,22 +15,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.beam.runners.dataflow.worker;
 
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
+import com.google.auto.value.AutoValue;
 
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Website_Test', 'Run Full Website Test',
-    'Full Website Test', this) {
+@AutoValue
+public abstract class ActiveMessageMetadata {
 
-      description('Test to validate the Beam website, including external links.')
+  public abstract String userStepName();
 
-      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 30, true, 'git-websites')
+  public abstract long startTime();
 
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':website:testWebsite -PdisableExternal=false')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
+  static ActiveMessageMetadata create(String userStepName, Long startTime) {
+    return new AutoValue_ActiveMessageMetadata(userStepName, startTime);
+  }
+}
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java
index d1de0833d87d..7d45295b2d8c 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContext.java
@@ -24,8 +24,11 @@
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.IntSummaryStatistics;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Optional;
 import org.apache.beam.runners.core.NullSideInputReader;
 import org.apache.beam.runners.core.SideInputReader;
 import org.apache.beam.runners.core.StepContext;
@@ -46,6 +49,7 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.io.Closer;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.DateTimeUtils.MillisProvider;
 import org.joda.time.Instant;
 
 /** Execution context for the Dataflow worker. */
@@ -245,6 +249,16 @@ public static class DataflowExecutionStateTracker extends ExecutionStateTracker
     private final ContextActivationObserverRegistry contextActivationObserverRegistry;
     private final String workItemId;
 
+    /**
+     * Metadata on the message whose processing is currently being managed by this tracker. If no
+     * message is actively being processed, activeMessageMetadata will be null.
+     */
+    @Nullable private ActiveMessageMetadata activeMessageMetadata = null;
+
+    private final MillisProvider clock = System::currentTimeMillis;
+
+    private final Map<String, IntSummaryStatistics> processingTimesByStep = new HashMap<>();
+
     public DataflowExecutionStateTracker(
         ExecutionStateSampler sampler,
         DataflowOperationContext.DataflowExecutionState otherState,
@@ -287,17 +301,32 @@ protected void takeSampleOnce(long millisSinceLastSample) {
       super.takeSampleOnce(millisSinceLastSample);
     }
 
+    /**
+     * Enter a new state on the tracker. If the new state is a Dataflow processing state, tracks the
+     * activeMessageMetadata with the start time of the new state.
+     */
     @Override
     public Closeable enterState(ExecutionState newState) {
       Closeable baseCloseable = super.enterState(newState);
       final boolean isDataflowProcessElementState =
           newState.isProcessElementState && newState instanceof DataflowExecutionState;
       if (isDataflowProcessElementState) {
-        elementExecutionTracker.enter(((DataflowExecutionState) newState).getStepName());
+        DataflowExecutionState newDFState = (DataflowExecutionState) newState;
+        if (newDFState.getStepName() != null && newDFState.getStepName().userName() != null) {
+          if (this.activeMessageMetadata != null) {
+            recordActiveMessageInProcessingTimesMap();
+          }
+          this.activeMessageMetadata =
+              ActiveMessageMetadata.create(newDFState.getStepName().userName(), clock.getMillis());
+        }
+        elementExecutionTracker.enter(newDFState.getStepName());
       }
 
       return () -> {
         if (isDataflowProcessElementState) {
+          if (this.activeMessageMetadata != null) {
+            recordActiveMessageInProcessingTimesMap();
+          }
           elementExecutionTracker.exit();
         }
         baseCloseable.close();
@@ -307,5 +336,35 @@ public Closeable enterState(ExecutionState newState) {
     public String getWorkItemId() {
       return this.workItemId;
     }
+
+    public Optional<ActiveMessageMetadata> getActiveMessageMetadata() {
+      return Optional.ofNullable(activeMessageMetadata);
+    }
+
+    public Map<String, IntSummaryStatistics> getProcessingTimesByStepCopy() {
+      Map<String, IntSummaryStatistics> processingTimesCopy = processingTimesByStep;
+      return processingTimesCopy;
+    }
+
+    /**
+     * Transitions the metadata for the currently active message to an entry in the completed
+     * processing times map. Sets the activeMessageMetadata to null after the entry has been
+     * recorded.
+     */
+    private void recordActiveMessageInProcessingTimesMap() {
+      if (this.activeMessageMetadata == null) {
+        return;
+      }
+      this.processingTimesByStep.compute(
+          this.activeMessageMetadata.userStepName(),
+          (k, v) -> {
+            if (v == null) {
+              v = new IntSummaryStatistics();
+            }
+            v.accept((int) (System.currentTimeMillis() - this.activeMessageMetadata.startTime()));
+            return v;
+          });
+      this.activeMessageMetadata = null;
+    }
   }
 }
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSampler.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSampler.java
new file mode 100644
index 000000000000..61b7e01e49a0
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSampler.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.worker;
+
+import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.HashMap;
+import java.util.IntSummaryStatistics;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.beam.runners.core.metrics.ExecutionStateSampler;
+import org.apache.beam.runners.core.metrics.ExecutionStateTracker;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowExecutionStateTracker;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
+import org.joda.time.DateTimeUtils.MillisProvider;
+
+public final class DataflowExecutionStateSampler extends ExecutionStateSampler {
+
+  private static final MillisProvider SYSTEM_MILLIS_PROVIDER = System::currentTimeMillis;
+  private static final DataflowExecutionStateSampler INSTANCE =
+      new DataflowExecutionStateSampler(SYSTEM_MILLIS_PROVIDER);
+
+  private final ConcurrentHashMap<String, DataflowExecutionStateTracker> activeTrackersByWorkId =
+      new ConcurrentHashMap<>();
+  private final ConcurrentHashMap<String, Map<String, IntSummaryStatistics>>
+      completedProcessingMetrics = new ConcurrentHashMap<>();
+
+  public static DataflowExecutionStateSampler instance() {
+    return INSTANCE;
+  }
+
+  @VisibleForTesting
+  public static DataflowExecutionStateSampler newForTest(MillisProvider clock) {
+    return new DataflowExecutionStateSampler(checkNotNull(clock));
+  }
+
+  public DataflowExecutionStateSampler(MillisProvider clock) {
+    super(clock);
+  }
+
+  @Override
+  public void addTracker(ExecutionStateTracker tracker) {
+    if (!(tracker instanceof DataflowExecutionStateTracker)) {
+      return;
+    }
+    DataflowExecutionStateTracker dfTracker = (DataflowExecutionStateTracker) tracker;
+    this.activeTrackersByWorkId.put(dfTracker.getWorkItemId(), dfTracker);
+  }
+
+  private static Map<String, IntSummaryStatistics> mergeStepStatsMaps(
+      Map<String, IntSummaryStatistics> map1, Map<String, IntSummaryStatistics> map2) {
+    for (Entry<String, IntSummaryStatistics> steps : map2.entrySet()) {
+      map1.compute(
+          steps.getKey(),
+          (k, v) -> {
+            if (v == null) {
+              return steps.getValue();
+            }
+            v.combine(steps.getValue());
+            return v;
+          });
+    }
+    return map1;
+  }
+
+  @Override
+  public void removeTracker(ExecutionStateTracker tracker) {
+    if (!(tracker instanceof DataflowExecutionContext.DataflowExecutionStateTracker)) {
+      return;
+    }
+    DataflowExecutionStateTracker dfTracker = (DataflowExecutionStateTracker) tracker;
+    completedProcessingMetrics.put(
+        dfTracker.getWorkItemId(), dfTracker.getProcessingTimesByStepCopy());
+    activeTrackersByWorkId.remove(dfTracker.getWorkItemId());
+
+    // Attribute any remaining time since the last sampling while removing the tracker.
+    //
+    // There is a race condition here; if sampling happens in the time between when we remove the
+    // tracker from activeTrackers and read the lastSampleTicks value, the sampling time will
+    // be lost for the tracker being removed. This is acceptable as sampling is already an
+    // approximation of actual execution time.
+    long millisSinceLastSample = clock.getMillis() - this.lastSampleTimeMillis;
+    if (millisSinceLastSample > 0) {
+      tracker.takeSample(millisSinceLastSample);
+    }
+  }
+
+  @Override
+  public void doSampling(long millisSinceLastSample) {
+    for (DataflowExecutionStateTracker tracker : activeTrackersByWorkId.values()) {
+      tracker.takeSample(millisSinceLastSample);
+    }
+  }
+
+  public Optional<ActiveMessageMetadata> getActiveMessageMetadataForWorkId(String workId) {
+    if (activeTrackersByWorkId.containsKey(workId)) {
+      return Optional.ofNullable(
+          activeTrackersByWorkId.get(workId).getActiveMessageMetadata().orElse(null));
+    }
+    return Optional.ofNullable(null);
+  }
+
+  public Map<String, IntSummaryStatistics> getProcessingDistributionsForWorkId(String workId) {
+    if (!activeTrackersByWorkId.containsKey(workId)) {
+      if (completedProcessingMetrics.containsKey(workId)) {
+        return completedProcessingMetrics.get(workId);
+      }
+      return new HashMap<>();
+    }
+    DataflowExecutionStateTracker tracker = activeTrackersByWorkId.get(workId);
+    return mergeStepStatsMaps(
+        completedProcessingMetrics.getOrDefault(workId, new HashMap<>()),
+        tracker.getProcessingTimesByStepCopy());
+  }
+
+  public void resetForWorkId(String workId) {
+    completedProcessingMetrics.remove(workId);
+  }
+}
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RemoveSafeDeltaCounterCell.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RemoveSafeDeltaCounterCell.java
new file mode 100644
index 000000000000..cf68de42acb2
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/RemoveSafeDeltaCounterCell.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.worker;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.beam.runners.core.metrics.CounterCell;
+import org.apache.beam.runners.core.metrics.DirtyState;
+import org.apache.beam.runners.core.metrics.MetricCell;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.MetricName;
+
+/**
+ * Version of {@link CounterCell} supporting multi-thread safe mutations and extraction of delta
+ * values. {@link RemoveSafeDeltaCounterCell} allows safe deletions from the underlying {@code
+ * countersMap} through the {@code deleteIfZero} method.
+ */
+public class RemoveSafeDeltaCounterCell implements Counter, MetricCell<Long> {
+
+  private final MetricName metricName;
+  /**
+   * This class does not own {@code countersMap} and only operates on a single key in the map
+   * specified by {@code metricName}. These opeations include the {@link Counter} interface along
+   * with the {@code deleteIfZero} method.
+   */
+  private final ConcurrentHashMap<MetricName, AtomicLong> countersMap;
+
+  /**
+   * @param metricName Specifies which metric this counter refers to.
+   * @param countersMap The underlying {@code map} used to store this metric.
+   */
+  public RemoveSafeDeltaCounterCell(
+      MetricName metricName, ConcurrentHashMap<MetricName, AtomicLong> countersMap) {
+    this.metricName = metricName;
+    this.countersMap = countersMap;
+  }
+
+  @Override
+  public void reset() {
+    countersMap.computeIfPresent(
+        metricName,
+        (unusedName, value) -> {
+          value.set(0);
+          return value;
+        });
+  }
+
+  @Override
+  public void inc(long n) {
+    countersMap.compute(
+        metricName,
+        (name, value) -> {
+          if (value == null) {
+            return new AtomicLong(n);
+          }
+          value.addAndGet(n);
+          return value;
+        });
+  }
+
+  @Override
+  public void inc() {
+    inc(1);
+  }
+
+  @Override
+  public void dec() {
+    inc(-1);
+  }
+
+  @Override
+  public void dec(long n) {
+    inc(-1 * n);
+  }
+
+  @Override
+  public MetricName getName() {
+    return metricName;
+  }
+
+  @Override
+  public DirtyState getDirty() {
+    throw new UnsupportedOperationException(
+        String.format("%s doesn't support the getDirty", getClass().getSimpleName()));
+  }
+
+  @Override
+  public Long getCumulative() {
+    throw new UnsupportedOperationException("getCumulative is not supported by Streaming Metrics");
+  }
+
+  /** Remove the metric from the {@code countersMap} if the the metric is zero valued. */
+  @SuppressWarnings(
+      "nullness") // computeIfPresent's remapping function can return null to remove the entry.
+  public void deleteIfZero() {
+    countersMap.computeIfPresent(
+        metricName,
+        (name, value) -> {
+          return value.get() == 0L ? null : value;
+        });
+  }
+}
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java
index 222d12c93bdf..4628f1cbeb03 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java
@@ -57,13 +57,13 @@
 import java.util.function.Supplier;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import org.apache.beam.runners.core.metrics.ExecutionStateSampler;
 import org.apache.beam.runners.core.metrics.MetricsLogger;
 import org.apache.beam.runners.dataflow.DataflowRunner;
 import org.apache.beam.runners.dataflow.internal.CustomSources;
 import org.apache.beam.runners.dataflow.options.DataflowWorkerHarnessOptions;
 import org.apache.beam.runners.dataflow.util.CloudObject;
 import org.apache.beam.runners.dataflow.util.CloudObjects;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionStateSampler;
 import org.apache.beam.runners.dataflow.worker.DataflowSystemMetrics.StreamingSystemCounterNames;
 import org.apache.beam.runners.dataflow.worker.apiary.FixMultiOutputInfosOnParDoInstructions;
 import org.apache.beam.runners.dataflow.worker.counters.Counter;
@@ -287,6 +287,8 @@ public class StreamingDataflowWorker {
   // Possibly overridden by streaming engine config.
   private int maxWorkItemCommitBytes = Integer.MAX_VALUE;
 
+  private final DataflowExecutionStateSampler sampler = DataflowExecutionStateSampler.instance();
+
   @VisibleForTesting
   StreamingDataflowWorker(
       List<MapTask> mapTasks,
@@ -519,12 +521,8 @@ private static void sleep(int millis) {
   }
 
   /** Sets the stage name and workId of the current Thread for logging. */
-  private static void setUpWorkLoggingContext(Windmill.WorkItem workItem, String computationId) {
-    String workIdBuilder =
-        Long.toHexString(workItem.getShardingKey())
-            + '-'
-            + Long.toHexString(workItem.getWorkToken());
-    DataflowWorkerLoggingMDC.setWorkId(workIdBuilder);
+  private static void setUpWorkLoggingContext(String workId, String computationId) {
+    DataflowWorkerLoggingMDC.setWorkId(workId);
     DataflowWorkerLoggingMDC.setStageName(computationId);
   }
 
@@ -585,7 +583,7 @@ public void start() {
     memoryMonitorThread.start();
     dispatchThread.start();
     commitThread.start();
-    ExecutionStateSampler.instance().start();
+    sampler.start();
 
     // Periodically report workers counters and other updates.
     globalWorkerUpdatesTimer = executorSupplier.apply("GlobalWorkerUpdatesTimer");
@@ -961,7 +959,7 @@ private void process(
     }
     work.setState(State.PROCESSING);
 
-    setUpWorkLoggingContext(workItem, computationId);
+    setUpWorkLoggingContext(work.getLatencyTrackingId(), computationId);
 
     LOG.debug("Starting processing for {}:\n{}", computationId, work);
 
@@ -1007,7 +1005,7 @@ private void process(
             (InstructionOutputNode) Iterables.getOnlyElement(mapTaskNetwork.successors(readNode));
         DataflowExecutionContext.DataflowExecutionStateTracker executionStateTracker =
             new DataflowExecutionContext.DataflowExecutionStateTracker(
-                ExecutionStateSampler.instance(),
+                sampler,
                 stageInfo
                     .executionStateRegistry()
                     .getState(
@@ -1017,7 +1015,7 @@ private void process(
                         ScopedProfiler.INSTANCE.emptyScope()),
                 stageInfo.deltaCounters(),
                 options,
-                computationId);
+                work.getLatencyTrackingId());
         StreamingModeExecutionContext context =
             new StreamingModeExecutionContext(
                 pendingDeltaCounters,
@@ -1181,7 +1179,8 @@ public void close() {
 
       // Add the output to the commit queue.
       work.setState(State.COMMIT_QUEUED);
-      outputBuilder.addAllPerWorkItemLatencyAttributions(work.getLatencyAttributions());
+      outputBuilder.addAllPerWorkItemLatencyAttributions(
+          work.getLatencyAttributions(false, work.getLatencyTrackingId(), sampler));
 
       WorkItemCommitRequest commitRequest = outputBuilder.build();
       int byteLimit = maxWorkItemCommitBytes;
@@ -1317,6 +1316,7 @@ public void close() {
         stageInfo.timerProcessingMsecs().addValue(processingTimeMsecs);
       }
 
+      sampler.resetForWorkId(work.getLatencyTrackingId());
       DataflowWorkerLoggingMDC.setWorkId(null);
       DataflowWorkerLoggingMDC.setStageName(null);
     }
@@ -1920,9 +1920,9 @@ private void refreshActiveWork() {
     for (Map.Entry<String, ComputationState> entry : computationMap.entrySet()) {
       if (windmillServiceEnabled
           && DataflowRunner.hasExperiment(options, "send_new_heartbeat_requests")) {
-        heartbeats.put(entry.getKey(), entry.getValue().getKeyHeartbeats(refreshDeadline));
+        heartbeats.put(entry.getKey(), entry.getValue().getKeyHeartbeats(refreshDeadline, sampler));
       } else {
-        active.put(entry.getKey(), entry.getValue().getKeysToRefresh(refreshDeadline));
+        active.put(entry.getKey(), entry.getValue().getKeysToRefresh(refreshDeadline, sampler));
       }
     }
 
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java
index 875a2d649ece..df1599f47189 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainer.java
@@ -21,6 +21,8 @@
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
 import javax.annotation.Nonnull;
 import org.apache.beam.runners.core.metrics.DistributionData;
 import org.apache.beam.runners.core.metrics.GaugeCell;
@@ -56,8 +58,7 @@ public class StreamingStepMetricsContainer implements MetricsContainer {
   private MetricsMap<MetricName, DeltaCounterCell> counters =
       new MetricsMap<>(DeltaCounterCell::new);
 
-  private MetricsMap<MetricName, DeltaCounterCell> perWorkerCounters =
-      new MetricsMap<>(DeltaCounterCell::new);
+  private ConcurrentHashMap<MetricName, AtomicLong> perWorkerCounters = new ConcurrentHashMap<>();
 
   private MetricsMap<MetricName, GaugeCell> gauges = new MetricsMap<>(GaugeCell::new);
 
@@ -88,7 +89,7 @@ public Counter getCounter(MetricName metricName) {
   @Override
   public Counter getPerWorkerCounter(MetricName metricName) {
     if (enablePerWorkerMetrics) {
-      return perWorkerCounters.get(metricName);
+      return new RemoveSafeDeltaCounterCell(metricName, perWorkerCounters);
     } else {
       return MetricsContainer.super.getPerWorkerCounter(metricName);
     }
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java
index 3f3d09220217..1a9e4a4ba735 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkState.java
@@ -32,6 +32,7 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.GuardedBy;
 import javax.annotation.concurrent.ThreadSafe;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionStateSampler;
 import org.apache.beam.runners.dataflow.worker.windmill.Windmill;
 import org.apache.beam.runners.dataflow.worker.windmill.Windmill.HeartbeatRequest;
 import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest;
@@ -225,14 +226,17 @@ private synchronized ImmutableMap<ShardedKey, Long> getStuckCommitsAt(
     return stuckCommits.build();
   }
 
-  synchronized ImmutableList<KeyedGetDataRequest> getKeysToRefresh(Instant refreshDeadline) {
+  synchronized ImmutableList<KeyedGetDataRequest> getKeysToRefresh(
+      Instant refreshDeadline, DataflowExecutionStateSampler sampler) {
     return activeWork.entrySet().stream()
-        .flatMap(entry -> toKeyedGetDataRequestStream(entry, refreshDeadline))
+        .flatMap(entry -> toKeyedGetDataRequestStream(entry, refreshDeadline, sampler))
         .collect(toImmutableList());
   }
 
   private static Stream<KeyedGetDataRequest> toKeyedGetDataRequestStream(
-      Entry<ShardedKey, Deque<Work>> shardedKeyAndWorkQueue, Instant refreshDeadline) {
+      Entry<ShardedKey, Deque<Work>> shardedKeyAndWorkQueue,
+      Instant refreshDeadline,
+      DataflowExecutionStateSampler sampler) {
     ShardedKey shardedKey = shardedKeyAndWorkQueue.getKey();
     Deque<Work> workQueue = shardedKeyAndWorkQueue.getValue();
 
@@ -244,18 +248,21 @@ private static Stream<KeyedGetDataRequest> toKeyedGetDataRequestStream(
                     .setKey(shardedKey.key())
                     .setShardingKey(shardedKey.shardingKey())
                     .setWorkToken(work.getWorkItem().getWorkToken())
-                    .addAllLatencyAttribution(work.getLatencyAttributions())
+                    .addAllLatencyAttribution(
+                        work.getLatencyAttributions(true, work.getLatencyTrackingId(), sampler))
                     .build());
   }
 
-  synchronized ImmutableList<HeartbeatRequest> getKeyHeartbeats(Instant refreshDeadline) {
+  synchronized ImmutableList<HeartbeatRequest> getKeyHeartbeats(
+      Instant refreshDeadline, DataflowExecutionStateSampler sampler) {
     return activeWork.entrySet().stream()
-        .flatMap(entry -> toHeartbeatRequestStream(entry, refreshDeadline))
+        .flatMap(entry -> toHeartbeatRequestStream(entry, refreshDeadline, sampler))
         .collect(toImmutableList());
   }
 
   private static Stream<HeartbeatRequest> toHeartbeatRequestStream(
-      Entry<ShardedKey, Deque<Work>> shardedKeyAndWorkQueue, Instant refreshDeadline) {
+      Entry<ShardedKey, Deque<Work>> shardedKeyAndWorkQueue, Instant refreshDeadline,
+      DataflowExecutionStateSampler sampler) {
     ShardedKey shardedKey = shardedKeyAndWorkQueue.getKey();
     Deque<Work> workQueue = shardedKeyAndWorkQueue.getValue();
 
@@ -269,7 +276,8 @@ private static Stream<HeartbeatRequest> toHeartbeatRequestStream(
                     .setShardingKey(shardedKey.shardingKey())
                     .setWorkToken(work.getWorkItem().getWorkToken())
                     .setCacheToken(work.getWorkItem().getCacheToken())
-                    .addAllLatencyAttribution(work.getLatencyAttributions())
+                    .addAllLatencyAttribution(
+                        work.getLatencyAttributions(true, work.getLatencyTrackingId(), sampler))
                     .build());
   }
 
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java
index 747806f2f18e..948493105a23 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/ComputationState.java
@@ -19,14 +19,15 @@
 
 import com.google.api.services.dataflow.model.MapTask;
 import java.io.PrintWriter;
-import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionStateSampler;
 import org.apache.beam.runners.dataflow.worker.util.BoundedQueueExecutor;
-import org.apache.beam.runners.dataflow.worker.windmill.Windmill;
+import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest;
 import org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
 import org.joda.time.Instant;
 
@@ -124,13 +125,15 @@ private void forceExecute(Work work) {
   }
 
   /** Adds any work started before the refreshDeadline to the GetDataRequest builder. */
-  public List<Windmill.KeyedGetDataRequest> getKeysToRefresh(Instant refreshDeadline) {
-    return activeWorkState.getKeysToRefresh(refreshDeadline);
+  public ImmutableList<KeyedGetDataRequest> getKeysToRefresh(
+      Instant refreshDeadline, DataflowExecutionStateSampler sampler) {
+    return activeWorkState.getKeysToRefresh(refreshDeadline, sampler);
   }
 
   /** Gets HeartbeatRequests for any work started before refreshDeadline. */
-  public List<Windmill.HeartbeatRequest> getKeyHeartbeats(Instant refreshDeadline) {
-    return activeWorkState.getKeyHeartbeats(refreshDeadline);
+  public List<Windmill.HeartbeatRequest> getKeyHeartbeats(
+      Instant refreshDeadline, DataflowExecutionStateSampler sampler) {
+    return activeWorkState.getKeyHeartbeats(refreshDeadline, sampler);
   }
 
   public void printActiveWork(PrintWriter writer) {
diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java
index a0e5a13119c1..bab2b9893fa2 100644
--- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java
+++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/Work.java
@@ -21,12 +21,22 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumMap;
+import java.util.IntSummaryStatistics;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import javax.annotation.concurrent.NotThreadSafe;
+import org.apache.beam.runners.dataflow.worker.ActiveMessageMetadata;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionStateSampler;
 import org.apache.beam.runners.dataflow.worker.windmill.Windmill;
+import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution;
+import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.ActiveLatencyBreakdown;
+import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.ActiveLatencyBreakdown.ActiveElementMetadata;
+import org.apache.beam.runners.dataflow.worker.windmill.Windmill.LatencyAttribution.ActiveLatencyBreakdown.Distribution;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 
@@ -93,6 +103,14 @@ public Instant getStateStartTime() {
     return currentState.startTime();
   }
 
+  public String getLatencyTrackingId() {
+    StringBuilder workIdBuilder = new StringBuilder(33);
+    workIdBuilder.append(Long.toHexString(workItem.getShardingKey()));
+    workIdBuilder.append('-');
+    workIdBuilder.append(Long.toHexString(workItem.getWorkToken()));
+    return workIdBuilder.toString();
+  }
+
   private void recordGetWorkStreamLatencies(
       Collection<Windmill.LatencyAttribution> getWorkStreamLatencies) {
     for (Windmill.LatencyAttribution latency : getWorkStreamLatencies) {
@@ -101,7 +119,8 @@ private void recordGetWorkStreamLatencies(
     }
   }
 
-  public Collection<Windmill.LatencyAttribution> getLatencyAttributions() {
+  public ImmutableList<LatencyAttribution> getLatencyAttributions(
+      boolean isHeartbeat, String workId, DataflowExecutionStateSampler sampler) {
     List<Windmill.LatencyAttribution> list = new ArrayList<>();
     for (Windmill.LatencyAttribution.State state : Windmill.LatencyAttribution.State.values()) {
       Duration duration = totalDurationPerState.getOrDefault(state, Duration.ZERO);
@@ -111,13 +130,54 @@ public Collection<Windmill.LatencyAttribution> getLatencyAttributions() {
       if (duration.equals(Duration.ZERO)) {
         continue;
       }
-      list.add(
-          Windmill.LatencyAttribution.newBuilder()
-              .setState(state)
-              .setTotalDurationMillis(duration.getMillis())
-              .build());
+      LatencyAttribution.Builder laBuilder = Windmill.LatencyAttribution.newBuilder();
+      if (state == LatencyAttribution.State.ACTIVE) {
+        laBuilder = addActiveLatencyBreakdownToBuilder(isHeartbeat, laBuilder, workId, sampler);
+      }
+      Windmill.LatencyAttribution la =
+          laBuilder.setState(state).setTotalDurationMillis(duration.getMillis()).build();
+      list.add(la);
+    }
+    return ImmutableList.copyOf(list);
+  }
+
+  private static LatencyAttribution.Builder addActiveLatencyBreakdownToBuilder(
+      boolean isHeartbeat,
+      LatencyAttribution.Builder builder,
+      String workId,
+      DataflowExecutionStateSampler sampler) {
+    if (isHeartbeat) {
+      ActiveLatencyBreakdown.Builder stepBuilder = ActiveLatencyBreakdown.newBuilder();
+      Optional<ActiveMessageMetadata> activeMessage =
+          sampler.getActiveMessageMetadataForWorkId(workId);
+      if (!activeMessage.isPresent()) {
+        return builder;
+      }
+      stepBuilder.setUserStepName(activeMessage.get().userStepName());
+      ActiveElementMetadata.Builder activeElementBuilder = ActiveElementMetadata.newBuilder();
+      activeElementBuilder.setProcessingTimeMillis(
+          System.currentTimeMillis() - activeMessage.get().startTime());
+      stepBuilder.setActiveMessageMetadata(activeElementBuilder);
+      builder.addActiveLatencyBreakdown(stepBuilder.build());
+      return builder;
+    }
+
+    Map<String, IntSummaryStatistics> processingDistributions =
+        sampler.getProcessingDistributionsForWorkId(workId);
+    for (Entry<String, IntSummaryStatistics> entry : processingDistributions.entrySet()) {
+      ActiveLatencyBreakdown.Builder stepBuilder = ActiveLatencyBreakdown.newBuilder();
+      stepBuilder.setUserStepName(entry.getKey());
+      Distribution.Builder distributionBuilder =
+          Distribution.newBuilder()
+              .setCount(entry.getValue().getCount())
+              .setMin(entry.getValue().getMin())
+              .setMax(entry.getValue().getMax())
+              .setMean((long) entry.getValue().getAverage())
+              .setSum(entry.getValue().getSum());
+      stepBuilder.setProcessingTimesDistribution(distributionBuilder.build());
+      builder.addActiveLatencyBreakdown(stepBuilder.build());
     }
-    return list;
+    return builder;
   }
 
   public boolean isFailed() {
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java
index b4f544129db6..e33412a19d94 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/BatchDataflowWorkerTest.java
@@ -50,6 +50,7 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.ArgumentCaptor;
@@ -61,6 +62,7 @@
 @RunWith(JUnit4.class)
 public class BatchDataflowWorkerTest {
 
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Rule public FastNanoClockAndSleeper clockAndSleeper = new FastNanoClockAndSleeper();
   @Mock WorkUnitClient mockWorkUnitClient;
   @Mock DataflowWorkProgressUpdater mockProgressUpdater;
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContextTest.java
index 7008dd4a4e97..01951c2f83e5 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContextTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionContextTest.java
@@ -17,14 +17,23 @@
  */
 package org.apache.beam.runners.dataflow.worker;
 
+import static org.apache.beam.runners.core.metrics.ExecutionStateTracker.PROCESS_STATE_NAME;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import com.google.auto.service.AutoService;
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.IntSummaryStatistics;
+import java.util.Map;
+import org.apache.beam.runners.core.metrics.ExecutionStateSampler;
 import org.apache.beam.runners.core.metrics.ExecutionStateTracker;
+import org.apache.beam.runners.dataflow.worker.StreamingModeExecutionContext.StreamingModeExecutionState;
+import org.apache.beam.runners.dataflow.worker.profiler.ScopedProfiler.NoopProfileScope;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.junit.Assert;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -111,4 +120,71 @@ public void testContextActivationObserverActivation() throws Exception {
     // AutoRegistrationClassNotActive class is not registered as registrar for the same is disabled.
     assertFalse(AutoRegistrationClassNotActive.WAS_CALLED);
   }
+
+  @Test
+  public void testDataflowExecutionStateTrackerRecordsActiveMessageMetadata() throws IOException {
+    DataflowExecutionContext.DataflowExecutionStateTracker tracker =
+        new DataflowExecutionContext.DataflowExecutionStateTracker(
+            ExecutionStateSampler.instance(), null, null, PipelineOptionsFactory.create(), "");
+    StreamingModeExecutionState state =
+        new StreamingModeExecutionState(
+            NameContextsForTests.nameContextForTest(),
+            PROCESS_STATE_NAME,
+            null,
+            NoopProfileScope.NOOP,
+            null);
+
+    Closeable closure = tracker.enterState(state);
+
+    // After entering a process state, we should have an active message tracked.
+    ActiveMessageMetadata expectedMetadata =
+        ActiveMessageMetadata.create(NameContextsForTests.nameContextForTest().userName(), 1l);
+    assertTrue(tracker.getActiveMessageMetadata().isPresent());
+    Assert.assertEquals(
+        expectedMetadata.userStepName(), tracker.getActiveMessageMetadata().get().userStepName());
+
+    closure.close();
+
+    // Once the state closes, the active message should get cleared.
+    assertFalse(tracker.getActiveMessageMetadata().isPresent());
+  }
+
+  @Test
+  public void testDataflowExecutionStateTrackerRecordsCompletedProcessingTimes()
+      throws IOException {
+    DataflowExecutionContext.DataflowExecutionStateTracker tracker =
+        new DataflowExecutionContext.DataflowExecutionStateTracker(
+            ExecutionStateSampler.instance(), null, null, PipelineOptionsFactory.create(), "");
+
+    // Enter a processing state
+    StreamingModeExecutionState state =
+        new StreamingModeExecutionState(
+            NameContextsForTests.nameContextForTest(),
+            PROCESS_STATE_NAME,
+            null,
+            NoopProfileScope.NOOP,
+            null);
+    tracker.enterState(state);
+    // Enter a new processing state
+    StreamingModeExecutionState newState =
+        new StreamingModeExecutionState(
+            NameContextsForTests.nameContextForTest(),
+            PROCESS_STATE_NAME,
+            null,
+            NoopProfileScope.NOOP,
+            null);
+    tracker.enterState(newState);
+
+    // The first completed state should be recorded and the new state should be active.
+    Map<String, IntSummaryStatistics> gotProcessingTimes = tracker.getProcessingTimesByStepCopy();
+    Assert.assertEquals(1, gotProcessingTimes.size());
+    Assert.assertEquals(
+        new HashSet<>(Arrays.asList(NameContextsForTests.nameContextForTest().userName())),
+        gotProcessingTimes.keySet());
+    ActiveMessageMetadata expectedMetadata =
+        ActiveMessageMetadata.create(NameContextsForTests.nameContextForTest().userName(), 1l);
+    assertTrue(tracker.getActiveMessageMetadata().isPresent());
+    Assert.assertEquals(
+        expectedMetadata.userStepName(), tracker.getActiveMessageMetadata().get().userStepName());
+  }
 }
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSamplerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSamplerTest.java
new file mode 100644
index 000000000000..b772952b74c3
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowExecutionStateSamplerTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.worker;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.IntSummaryStatistics;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowExecutionStateTracker;
+import org.apache.beam.runners.dataflow.worker.counters.NameContext;
+import org.joda.time.DateTimeUtils.MillisProvider;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DataflowExecutionStateSamplerTest {
+
+  private MillisProvider clock;
+  private DataflowExecutionStateSampler sampler;
+
+  @Before
+  public void setUp() {
+    clock = mock(MillisProvider.class);
+    sampler = DataflowExecutionStateSampler.newForTest(clock);
+  }
+
+  private final TestOperationContext.TestDataflowExecutionState step1act1 =
+      new TestOperationContext.TestDataflowExecutionState(
+          createNameContext("test-stage1"), "activity1");
+
+  private NameContext createNameContext(String userName) {
+    return NameContext.create("", "", "", userName);
+  }
+
+  @Test
+  public void testAddTrackerRemoveTrackerActiveMessageMetadataGetsUpdated() {
+    String workId = "work-item-id1";
+    ActiveMessageMetadata testMetadata =
+        ActiveMessageMetadata.create(step1act1.getStepName().userName(), clock.getMillis());
+    DataflowExecutionStateTracker trackerMock = createMockTracker(workId);
+    when(trackerMock.getActiveMessageMetadata()).thenReturn(Optional.of(testMetadata));
+
+    sampler.addTracker(trackerMock);
+    assertThat(sampler.getActiveMessageMetadataForWorkId(workId).get(), equalTo(testMetadata));
+
+    sampler.removeTracker(trackerMock);
+    Assert.assertFalse(sampler.getActiveMessageMetadataForWorkId(workId).isPresent());
+  }
+
+  @Test
+  public void testRemoveTrackerCompletedProcessingTimesGetsUpdated() {
+    String workId = "work-item-id1";
+    Map<String, IntSummaryStatistics> testCompletedProcessingTimes = new HashMap<>();
+    testCompletedProcessingTimes.put("some-step", new IntSummaryStatistics());
+    DataflowExecutionStateTracker trackerMock = createMockTracker(workId);
+    when(trackerMock.getProcessingTimesByStepCopy()).thenReturn(testCompletedProcessingTimes);
+
+    sampler.addTracker(trackerMock);
+    sampler.removeTracker(trackerMock);
+
+    assertThat(
+        sampler.getProcessingDistributionsForWorkId(workId), equalTo(testCompletedProcessingTimes));
+  }
+
+  @Test
+  public void testGetCompletedProcessingTimesAndActiveMessageFromActiveTracker() {
+    String workId = "work-item-id1";
+    Map<String, IntSummaryStatistics> testCompletedProcessingTimes = new HashMap<>();
+    IntSummaryStatistics testSummaryStats = new IntSummaryStatistics();
+    testSummaryStats.accept(1);
+    testSummaryStats.accept(3);
+    testSummaryStats.accept(5);
+    testCompletedProcessingTimes.put("some-step", testSummaryStats);
+    ActiveMessageMetadata testMetadata =
+        ActiveMessageMetadata.create(step1act1.getStepName().userName(), clock.getMillis());
+    DataflowExecutionStateTracker trackerMock = createMockTracker(workId);
+    when(trackerMock.getActiveMessageMetadata()).thenReturn(Optional.of(testMetadata));
+    when(trackerMock.getProcessingTimesByStepCopy()).thenReturn(testCompletedProcessingTimes);
+
+    sampler.addTracker(trackerMock);
+
+    assertThat(sampler.getActiveMessageMetadataForWorkId(workId).get(), equalTo(testMetadata));
+    assertThat(
+        sampler.getProcessingDistributionsForWorkId(workId), equalTo(testCompletedProcessingTimes));
+  }
+
+  @Test
+  public void testResetForWorkIdClearsMaps() {
+    String workId1 = "work-item-id1";
+    String workId2 = "work-item-id2";
+    DataflowExecutionStateTracker tracker1Mock = createMockTracker(workId1);
+    DataflowExecutionStateTracker tracker2Mock = createMockTracker(workId2);
+
+    sampler.addTracker(tracker1Mock);
+    sampler.addTracker(tracker2Mock);
+
+    assertThat(
+        sampler.getActiveMessageMetadataForWorkId(workId1),
+        equalTo(tracker1Mock.getActiveMessageMetadata()));
+    assertThat(
+        sampler.getProcessingDistributionsForWorkId(workId1),
+        equalTo(tracker1Mock.getProcessingTimesByStepCopy()));
+    assertThat(
+        sampler.getActiveMessageMetadataForWorkId(workId2),
+        equalTo(tracker2Mock.getActiveMessageMetadata()));
+    assertThat(
+        sampler.getProcessingDistributionsForWorkId(workId2),
+        equalTo(tracker2Mock.getProcessingTimesByStepCopy()));
+
+    sampler.removeTracker(tracker1Mock);
+    sampler.removeTracker(tracker2Mock);
+    sampler.resetForWorkId(workId2);
+
+    assertThat(
+        sampler.getProcessingDistributionsForWorkId(workId1),
+        equalTo(tracker1Mock.getProcessingTimesByStepCopy()));
+    Assert.assertTrue(sampler.getProcessingDistributionsForWorkId(workId2).isEmpty());
+  }
+
+  private DataflowExecutionStateTracker createMockTracker(String workItemId) {
+    DataflowExecutionStateTracker trackerMock = mock(DataflowExecutionStateTracker.class);
+    when(trackerMock.getWorkItemId()).thenReturn(workItemId);
+    return trackerMock;
+  }
+}
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java
index 3c63f3cc19d2..5329fb0f601c 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/DataflowWorkUnitClientTest.java
@@ -50,6 +50,7 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -60,6 +61,8 @@
 /** Unit tests for {@link DataflowWorkUnitClient}. */
 @RunWith(JUnit4.class)
 public class DataflowWorkUnitClientTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
+
   private static final Logger LOG = LoggerFactory.getLogger(DataflowWorkUnitClientTest.class);
   private static final String PROJECT_ID = "TEST_PROJECT_ID";
   private static final String JOB_ID = "TEST_JOB_ID";
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java
index 0bc40a5f391f..5b6d7562057a 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/FakeWindmillServer.java
@@ -83,7 +83,7 @@ class FakeWindmillServer extends WindmillServerStub {
   private final ErrorCollector errorCollector;
   private final ConcurrentHashMap<Long, Consumer<Windmill.CommitStatus>> droppedStreamingCommits;
   private int commitsRequested = 0;
-  private int numGetDataRequests = 0;
+  private List<Windmill.GetDataRequest> getDataRequests = new ArrayList<>();
   private boolean isReady = true;
   private boolean dropStreamingCommits = false;
   private Consumer<List<Windmill.ComputationHeartbeatResponse>> processHeartbeatResponses;
@@ -159,7 +159,7 @@ private void validateGetDataRequest(Windmill.GetDataRequest request) {
   public Windmill.GetDataResponse getData(Windmill.GetDataRequest request) {
     LOG.info("getDataRequest: {}", request.toString());
     validateGetDataRequest(request);
-    ++numGetDataRequests;
+    getDataRequests.add(request);
     GetDataResponse response = dataToOffer.getOrDefault(request);
     LOG.debug("getDataResponse: {}", response.toString());
     return response;
@@ -472,7 +472,11 @@ public Windmill.Exception getException() throws InterruptedException {
   }
 
   public int numGetDataRequests() {
-    return numGetDataRequests;
+    return getDataRequests.size();
+  }
+
+  public List<Windmill.GetDataRequest> getGetDataRequests() {
+    return getDataRequests;
   }
 
   public ArrayList<Windmill.ReportStatsRequest> getStatsReceived() {
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RemoveSafeDeltaCounterCellTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RemoveSafeDeltaCounterCellTest.java
new file mode 100644
index 000000000000..419b733071d7
--- /dev/null
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/RemoveSafeDeltaCounterCellTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.dataflow.worker;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.beam.sdk.metrics.MetricName;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link RemoveSafeDeltaCounterCell}. */
+@RunWith(JUnit4.class)
+public class RemoveSafeDeltaCounterCellTest {
+
+  @Test
+  public void testRemoveSafeDeltaCounterCell_deleteZero() throws Exception {
+    ConcurrentHashMap<MetricName, AtomicLong> metric_map = new ConcurrentHashMap<>();
+    MetricName metric1 = MetricName.named("namespace", "name_1");
+
+    RemoveSafeDeltaCounterCell cell_1 = new RemoveSafeDeltaCounterCell(metric1, metric_map);
+    RemoveSafeDeltaCounterCell cell_2 = new RemoveSafeDeltaCounterCell(metric1, metric_map);
+
+    cell_2.inc(10);
+    assertThat(metric_map.get(metric1).get(), equalTo(10L));
+
+    cell_2.reset();
+    assertThat(metric_map.get(metric1).get(), equalTo(0L));
+    cell_2.deleteIfZero();
+
+    assertThat(metric_map.get(metric1), is(nullValue()));
+
+    // Incrementing a deleted counter will recreate it.
+    cell_1.inc(20);
+    assertThat(metric_map.get(metric1).get(), equalTo(20L));
+  }
+
+  @Test
+  public void testRemoveSafeDeltaCounterCell_deleteNonZero() throws Exception {
+    ConcurrentHashMap<MetricName, AtomicLong> metric_map = new ConcurrentHashMap<>();
+    MetricName metric1 = MetricName.named("namespace", "name_1");
+
+    RemoveSafeDeltaCounterCell cell_1 = new RemoveSafeDeltaCounterCell(metric1, metric_map);
+    cell_1.inc(10);
+
+    cell_1.deleteIfZero();
+    assertThat(metric_map.size(), equalTo(1));
+    assertThat(metric_map.get(metric1).get(), equalTo(10L));
+  }
+
+  @Test
+  public void testRemoveSafeDeltaCounterCell_multipleUpdates() throws Exception {
+    ConcurrentHashMap<MetricName, AtomicLong> metric_map = new ConcurrentHashMap<>();
+    MetricName metric1 = MetricName.named("namespace", "name_1");
+
+    RemoveSafeDeltaCounterCell cell_1 = new RemoveSafeDeltaCounterCell(metric1, metric_map);
+
+    cell_1.inc();
+    assertThat(metric_map.get(metric1).get(), equalTo(1L));
+
+    cell_1.inc(5);
+    assertThat(metric_map.get(metric1).get(), equalTo(6L));
+
+    cell_1.dec(10);
+    assertThat(metric_map.get(metric1).get(), equalTo(-4L));
+
+    cell_1.dec();
+    assertThat(metric_map.get(metric1).get(), equalTo(-5L));
+  }
+
+  @Test
+  public void testRemoveSafeDeltaCounterCell_resetNonExistant() throws Exception {
+    ConcurrentHashMap<MetricName, AtomicLong> metric_map = new ConcurrentHashMap<>();
+    MetricName metric1 = MetricName.named("namespace", "name_1");
+
+    RemoveSafeDeltaCounterCell cell_1 = new RemoveSafeDeltaCounterCell(metric1, metric_map);
+    assertThat(metric_map.size(), equalTo(0));
+
+    cell_1.reset();
+    assertThat(metric_map.size(), equalTo(0));
+  }
+}
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java
index f475d2d7701e..eba59addeeba 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java
@@ -25,6 +25,7 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasProperty;
 import static org.hamcrest.Matchers.lessThan;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -177,6 +178,7 @@
 import org.hamcrest.Matchers;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
+import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
@@ -198,7 +200,6 @@
 // released (2.11.0)
 @SuppressWarnings("unused")
 public class StreamingDataflowWorkerTest {
-
   private static final Logger LOG = LoggerFactory.getLogger(StreamingDataflowWorkerTest.class);
   private static final IntervalWindow DEFAULT_WINDOW =
       new IntervalWindow(new Instant(1234), Duration.millis(1000));
@@ -3322,7 +3323,8 @@ public void testLatencyAttributionProtobufsPopulated() {
     work.setState(Work.State.COMMITTING);
     clock.sleep(Duration.millis(60));
 
-    Iterator<LatencyAttribution> it = work.getLatencyAttributions().iterator();
+    Iterator<LatencyAttribution> it =
+        work.getLatencyAttributions(false, "", DataflowExecutionStateSampler.instance()).iterator();
     assertTrue(it.hasNext());
     LatencyAttribution lat = it.next();
     assertSame(State.QUEUED, lat.getState());
@@ -3535,12 +3537,19 @@ public void testLatencyAttributionPopulatedInCommitRequest() throws Exception {
 
     worker.stop();
 
-    assertEquals(
-        workItemCommitRequest.get((long) workToken).getPerWorkItemLatencyAttributions(0),
+    LatencyAttribution.Builder expectedActiveLA =
         LatencyAttribution.newBuilder()
             .setState(State.ACTIVE)
-            .setTotalDurationMillis(dofnWaitTimeMs)
-            .build());
+            .setTotalDurationMillis(dofnWaitTimeMs);
+    assertThat(
+        workItemCommitRequest.get((long) workToken).getPerWorkItemLatencyAttributions(0),
+        hasProperty("state", Matchers.equalTo(State.ACTIVE)));
+    assertThat(
+        workItemCommitRequest.get((long) workToken).getPerWorkItemLatencyAttributions(0),
+        hasProperty("totalDurationMillis", Matchers.equalTo(1000L)));
+    assertThat(
+        workItemCommitRequest.get((long) workToken).getPerWorkItemLatencyAttributions(0),
+        hasProperty("activeLatencyBreakdown"));
     if (streamingEngine) {
       // Initial fake latency provided to FakeWindmillServer when invoke receiveWork in
       // GetWorkStream().
@@ -3553,6 +3562,75 @@ public void testLatencyAttributionPopulatedInCommitRequest() throws Exception {
     }
   }
 
+  @Test
+  public void testDoFnLatencyBreakdownsReportedOnCommit() throws Exception {
+    List<ParallelInstruction> instructions =
+        Arrays.asList(
+            makeSourceInstruction(StringUtf8Coder.of()),
+            makeDoFnInstruction(new SlowDoFn(), 0, StringUtf8Coder.of()),
+            makeSinkInstruction(StringUtf8Coder.of(), 0));
+
+    FakeWindmillServer server = new FakeWindmillServer(errorCollector);
+    StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server);
+    options.setActiveWorkRefreshPeriodMillis(100);
+    StreamingDataflowWorker worker = makeWorker(instructions, options, true /* publishCounters */);
+    worker.start();
+
+    server.whenGetWorkCalled().thenReturn(makeInput(0, TimeUnit.MILLISECONDS.toMicros(0)));
+
+    Map<Long, Windmill.WorkItemCommitRequest> result = server.waitForAndGetCommits(1);
+    Windmill.WorkItemCommitRequest commit = result.get(0L);
+
+    Windmill.LatencyAttribution.Builder laBuilder =
+        LatencyAttribution.newBuilder().setState(State.ACTIVE).setTotalDurationMillis(100);
+    for (LatencyAttribution la : commit.getPerWorkItemLatencyAttributionsList()) {
+      if (la.getState() == State.ACTIVE) {
+        assertThat(la.getActiveLatencyBreakdownCount(), equalTo(1));
+        assertThat(
+            la.getActiveLatencyBreakdown(0).getUserStepName(), equalTo(DEFAULT_PARDO_USER_NAME));
+        Assert.assertTrue(la.getActiveLatencyBreakdown(0).hasProcessingTimesDistribution());
+        Assert.assertFalse(la.getActiveLatencyBreakdown(0).hasActiveMessageMetadata());
+      }
+    }
+
+    worker.stop();
+  }
+
+  @Test
+  public void testDoFnActiveMessageMetadataReportedOnHeartbeat() throws Exception {
+    if (!streamingEngine) {
+      return;
+    }
+    List<ParallelInstruction> instructions =
+        Arrays.asList(
+            makeSourceInstruction(StringUtf8Coder.of()),
+            makeDoFnInstruction(new SlowDoFn(), 0, StringUtf8Coder.of()),
+            makeSinkInstruction(StringUtf8Coder.of(), 0));
+
+    FakeWindmillServer server = new FakeWindmillServer(errorCollector);
+    StreamingDataflowWorkerOptions options = createTestingPipelineOptions(server);
+    options.setActiveWorkRefreshPeriodMillis(10);
+    StreamingDataflowWorker worker = makeWorker(instructions, options, true /* publishCounters */);
+    worker.start();
+
+    server.whenGetWorkCalled().thenReturn(makeInput(0, TimeUnit.MILLISECONDS.toMicros(0)));
+
+    Map<Long, Windmill.WorkItemCommitRequest> result = server.waitForAndGetCommits(1);
+
+    assertThat(server.numGetDataRequests(), greaterThan(0));
+    Windmill.GetDataRequest heartbeat = server.getGetDataRequests().get(2);
+
+    for (LatencyAttribution la :
+        heartbeat.getRequests(0).getRequests(0).getLatencyAttributionList()) {
+      if (la.getState() == State.ACTIVE) {
+        assertTrue(la.getActiveLatencyBreakdownCount() > 0);
+        assertTrue(la.getActiveLatencyBreakdown(0).hasActiveMessageMetadata());
+      }
+    }
+
+    worker.stop();
+  }
+
   @Test
   public void testLimitOnOutputBundleSize() throws Exception {
     // This verifies that ReadOperation, StreamingModeExecutionContext, and windmill sinks
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java
index 259f6d23ceb4..451ec649aa23 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java
@@ -73,7 +73,9 @@
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -83,7 +85,7 @@
 /** Tests for {@link StreamingModeExecutionContext}. */
 @RunWith(JUnit4.class)
 public class StreamingModeExecutionContextTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Mock private SideInputStateFetcher sideInputStateFetcher;
   @Mock private WindmillStateReader stateReader;
 
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java
index 3c121ab27f76..07c1080d8f23 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputDoFnRunnerTest.java
@@ -69,7 +69,9 @@
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -79,7 +81,7 @@
 /** Unit tests for {@link StreamingSideInputDoFnRunner}. */
 @RunWith(JUnit4.class)
 public class StreamingSideInputDoFnRunnerTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final FixedWindows WINDOW_FN = FixedWindows.of(Duration.millis(10));
 
   static TupleTag<String> mainOutputTag = new TupleTag<>();
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java
index a7196613fbb1..d4fee95ead54 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingSideInputFetcherTest.java
@@ -56,7 +56,9 @@
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -65,7 +67,7 @@
 /** Tests for {@link StreamingSideInputFetcher}. */
 @RunWith(JUnit4.class)
 public class StreamingSideInputFetcherTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final FixedWindows WINDOW_FN = FixedWindows.of(Duration.millis(10));
 
   static TupleTag<String> mainOutputTag = new TupleTag<>();
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java
index 9e6d45a2351b..9be65c198ac7 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingStepMetricsContainerTest.java
@@ -37,7 +37,9 @@
 import org.apache.beam.sdk.metrics.NoOpCounter;
 import org.apache.beam.sdk.metrics.NoOpHistogram;
 import org.apache.beam.sdk.util.HistogramData;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -47,7 +49,7 @@
   "rawtypes" // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class StreamingStepMetricsContainerTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private MetricsContainerRegistry registry = StreamingStepMetricsContainer.createRegistry();
 
   private MetricsContainer c1 = registry.getContainer("s1");
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java
index c1b134cafa1c..425b2140a962 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/logging/DataflowWorkerLoggingInitializerTest.java
@@ -55,6 +55,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.slf4j.LoggerFactory;
@@ -68,6 +69,7 @@
  */
 @RunWith(JUnit4.class)
 public class DataflowWorkerLoggingInitializerTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Rule public TemporaryFolder logFolder = new TemporaryFolder();
 
   @Rule public RestoreSystemProperties restoreProperties = new RestoreSystemProperties();
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java
index 12ae816de829..ea57f687fd95 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/ActiveWorkStateTest.java
@@ -33,6 +33,7 @@
 import java.util.Map;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import org.apache.beam.runners.dataflow.worker.DataflowExecutionStateSampler;
 import org.apache.beam.runners.dataflow.worker.streaming.ActiveWorkState.ActivateWorkResult;
 import org.apache.beam.runners.dataflow.worker.windmill.Windmill;
 import org.apache.beam.runners.dataflow.worker.windmill.Windmill.KeyedGetDataRequest;
@@ -41,13 +42,15 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class ActiveWorkStateTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private final WindmillStateCache.ForComputation computationStateCache =
       mock(WindmillStateCache.ForComputation.class);
   private Map<ShardedKey, Deque<Work>> readOnlyActiveWork;
@@ -251,7 +254,8 @@ public void testGetKeysToRefresh() {
     activeWorkState.activateWorkForKey(shardedKey1, freshWork);
     activeWorkState.activateWorkForKey(shardedKey2, refreshableWork2);
 
-    ImmutableList<KeyedGetDataRequest> requests = activeWorkState.getKeysToRefresh(refreshDeadline);
+    ImmutableList<KeyedGetDataRequest> requests =
+        activeWorkState.getKeysToRefresh(refreshDeadline, DataflowExecutionStateSampler.instance());
 
     ImmutableList<GetDataRequestKeyShardingKeyAndWorkToken> expected =
         ImmutableList.of(
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java
index b2d98fb0e954..4f035c88774c 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/WeightBoundedQueueTest.java
@@ -22,12 +22,15 @@
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class WeightBoundedQueueTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final int MAX_WEIGHT = 10;
 
   @Test
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/sideinput/SideInputStateFetcherTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/sideinput/SideInputStateFetcherTest.java
index daf814618791..1e188da2dd63 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/sideinput/SideInputStateFetcherTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/streaming/sideinput/SideInputStateFetcherTest.java
@@ -51,7 +51,9 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.Cache;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.cache.CacheBuilder;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.ArgumentCaptor;
@@ -63,6 +65,7 @@
 @SuppressWarnings("deprecation")
 @RunWith(JUnit4.class)
 public class SideInputStateFetcherTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final String STATE_FAMILY = "state";
 
   @Mock private MetricTrackingWindmillServerStub server;
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/WindmillStreamPoolTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/WindmillStreamPoolTest.java
index 264540531bf8..a2f5e71d04c3 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/WindmillStreamPoolTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/WindmillStreamPoolTest.java
@@ -30,12 +30,15 @@
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class WindmillStreamPoolTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final int DEFAULT_NUM_STREAMS = 10;
   private static final int NEW_STREAM_HOLDS = 2;
   private final ConcurrentHashMap<
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcGetWorkerMetadataStreamTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcGetWorkerMetadataStreamTest.java
index e3b07bf7aa4d..253d6ff3a48f 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcGetWorkerMetadataStreamTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcGetWorkerMetadataStreamTest.java
@@ -55,12 +55,14 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mockito;
 
 @RunWith(JUnit4.class)
 public class GrpcGetWorkerMetadataStreamTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final String IPV6_ADDRESS_1 = "2001:db8:0000:bac5:0000:0000:fed0:81a2";
   private static final String IPV6_ADDRESS_2 = "2001:db8:0000:bac5:0000:0000:fed0:82a3";
   private static final List<WorkerMetadataResponse.Endpoint> DIRECT_PATH_ENDPOINTS =
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcWindmillServerTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcWindmillServerTest.java
index a18503f277c0..d7025d58f873 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcWindmillServerTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcWindmillServerTest.java
@@ -96,6 +96,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ErrorCollector;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.slf4j.Logger;
@@ -107,7 +108,7 @@
   "rawtypes", // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class GrpcWindmillServerTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final Logger LOG = LoggerFactory.getLogger(GrpcWindmillServerTest.class);
   private static final int STREAM_CHUNK_SIZE = 2 << 20;
   private final MutableHandlerRegistry serviceRegistry = new MutableHandlerRegistry();
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/StreamingEngineClientTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/StreamingEngineClientTest.java
index 8a2c643a5b76..166b2cd0f031 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/StreamingEngineClientTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/StreamingEngineClientTest.java
@@ -64,13 +64,16 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.net.HostAndPort;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class StreamingEngineClientTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final WindmillServiceAddress DEFAULT_WINDMILL_SERVICE_ADDRESS =
       WindmillServiceAddress.create(HostAndPort.fromParts(WindmillChannelFactory.LOCALHOST, 443));
   private static final ImmutableMap<String, WorkerMetadataResponse.Endpoint> DEFAULT =
@@ -251,6 +254,7 @@ public void testScheduledBudgetRefresh() throws InterruptedException {
   }
 
   @Test
+  @Ignore("https://github.com/apache/beam/issues/28957") // stuck test
   public void testOnNewWorkerMetadata_correctlyRemovesStaleWindmillServers()
       throws InterruptedException {
     streamingEngineClient =
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/WindmillStreamSenderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/WindmillStreamSenderTest.java
index c8d2974f923d..5fd1814e511f 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/WindmillStreamSenderTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/WindmillStreamSenderTest.java
@@ -43,11 +43,13 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class WindmillStreamSenderTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final GetWorkRequest GET_WORK_REQUEST =
       GetWorkRequest.newBuilder().setClientId(1L).setJobId("job").setProjectId("project").build();
   @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java
index cc6633f1b704..35d01aaffb8a 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateCacheTest.java
@@ -35,14 +35,16 @@
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link org.apache.beam.runners.dataflow.worker.windmill.state.WindmillStateCache}. */
 @RunWith(JUnit4.class)
 public class WindmillStateCacheTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final String COMPUTATION = "computation";
   private static final long SHARDING_KEY = 123;
   private static final WindmillComputationKey COMPUTATION_KEY =
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java
index 8971c39ccaa1..d2590ceb8466 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateInternalsTest.java
@@ -98,7 +98,9 @@
 import org.joda.time.Instant;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.ArgumentCaptor;
@@ -112,7 +114,7 @@
   "rawtypes", // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class WindmillStateInternalsTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   public static final Range<Long> FULL_ORDERED_LIST_RANGE =
       Range.closedOpen(WindmillOrderedList.MIN_TS_MICROS, WindmillOrderedList.MAX_TS_MICROS);
   private static final StateNamespace NAMESPACE = new StateNamespaceForTest("ns");
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java
index 430e31ee04ff..7ef74639bb55 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/state/WindmillStateReaderTest.java
@@ -54,7 +54,9 @@
 import org.hamcrest.Matchers;
 import org.joda.time.Instant;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.ArgumentCaptor;
@@ -69,6 +71,7 @@
   "FutureReturnValueIgnored",
 })
 public class WindmillStateReaderTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final VarIntCoder INT_CODER = VarIntCoder.of();
 
   private static final String COMPUTATION = "computation";
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/EvenGetWorkBudgetDistributorTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/EvenGetWorkBudgetDistributorTest.java
index 14da55fe2389..54b605efbf3c 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/EvenGetWorkBudgetDistributorTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/EvenGetWorkBudgetDistributorTest.java
@@ -40,11 +40,13 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class EvenGetWorkBudgetDistributorTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
 
   private ManagedChannel inProcessChannel;
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetRefresherTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetRefresherTest.java
index fd85410cc91d..101e111cb657 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetRefresherTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetRefresherTest.java
@@ -21,13 +21,16 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
 
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mockito;
 
 @RunWith(JUnit4.class)
 public class GetWorkBudgetRefresherTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final int WAIT_BUFFER = 10;
   private final Runnable redistributeBudget = Mockito.mock(Runnable.class);
 
diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetTest.java
index 97789abaaa97..2d2806bef618 100644
--- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetTest.java
+++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/windmill/work/budget/GetWorkBudgetTest.java
@@ -19,12 +19,15 @@
 
 import static org.junit.Assert.assertEquals;
 
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class GetWorkBudgetTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Test
   public void testCreateWithNoBudget() {
diff --git a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
index 9283683ccf31..b05fc723ac3f 100644
--- a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
+++ b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
@@ -59,6 +59,21 @@ message KeyedMessageBundle {
 }
 
 message LatencyAttribution {
+  message ActiveLatencyBreakdown {
+    message Distribution {
+      optional int64 count = 1;
+      optional int64 sum = 2;
+      optional int64 min = 3;
+      optional int64 max = 4;
+      optional int64 mean = 5;
+    }
+    message ActiveElementMetadata {
+      optional int64 processing_time_millis = 1;
+    }
+    optional string user_step_name = 1;
+    optional Distribution processing_times_distribution = 2;
+    optional ActiveElementMetadata active_message_metadata = 3;
+  }
   enum State {
     UNKNOWN = 0;
     QUEUED = 1;
@@ -80,6 +95,7 @@ message LatencyAttribution {
   }
   optional State state = 1;
   optional int64 total_duration_millis = 2;
+  repeated ActiveLatencyBreakdown active_latency_breakdown = 3;
 }
 
 message PerStepNamespaceMetrics {
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java
index 824c2c78cc50..019028f5b936 100644
--- a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactory.java
@@ -232,6 +232,12 @@ private ImmutableList<EnvironmentCacheAndLock> createEnvironmentCaches(
               new CacheLoader<Environment, WrappedSdkHarnessClient>() {
                 @Override
                 public WrappedSdkHarnessClient load(Environment environment) throws Exception {
+                  // TODO(robertwb): Docker is the the safest fallback (if we are distributed)
+                  // but it would be good to have the ability to make a more intellegent choice
+                  // (e.g. in-process or loopback workers, especially if running locally).
+                  environment =
+                      Environments.resolveAnyOfEnvironment(
+                          environment, BeamUrns.getUrn(StandardEnvironments.Environments.DOCKER));
                   EnvironmentFactory.Provider environmentFactoryProvider =
                       environmentFactoryProviderMap.get(environment.getUrn());
                   ServerFactory serverFactory = environmentFactoryProvider.getServerFactory();
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java
index e9428cdc2c94..9a7cd2a9cff4 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/EmbeddedSdkHarness.java
@@ -37,8 +37,10 @@
 import org.apache.beam.sdk.fn.stream.OutboundObserverFactory;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.junit.Rule;
 import org.junit.rules.ExternalResource;
 import org.junit.rules.TestRule;
+import org.junit.rules.Timeout;
 
 /**
  * A {@link TestRule} which creates a {@link FnHarness} in a thread, services required for that
@@ -46,6 +48,7 @@
  * during test execution.
  */
 public class EmbeddedSdkHarness extends ExternalResource implements TestRule {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   public static EmbeddedSdkHarness create() {
     return new EmbeddedSdkHarness();
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java
index 187c68decc9d..8455b23fee90 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/GrpcContextHeaderAccessorProviderTest.java
@@ -40,6 +40,7 @@
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mockito;
@@ -48,6 +49,7 @@
 @RunWith(JUnit4.class)
 public class GrpcContextHeaderAccessorProviderTest {
   @Rule public GrpcCleanupRule cleanupRule = new GrpcCleanupRule().setTimeout(10, TimeUnit.SECONDS);
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @SuppressWarnings("unchecked")
   @Test
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java
index 6e19f9d6a7b4..0671fd1eb3ca 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/ServerFactoryTest.java
@@ -55,10 +55,11 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.Uninterruptibles;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 
 /** Tests for {@link ServerFactory}. */
 public class ServerFactoryTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final BeamFnApi.Elements CLIENT_DATA =
       BeamFnApi.Elements.newBuilder()
           .addData(BeamFnApi.Elements.Data.newBuilder().setInstructionId("1"))
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java
index bcce5f55a994..b7b7b1cfa58e 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalServiceTest.java
@@ -40,6 +40,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -51,6 +52,7 @@ public class ArtifactRetrievalServiceTest {
   private Path stagingDir;
   @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
   @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Before
   public void setUp() throws Exception {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java
index 3f80c97f4336..51c0d99e5933 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/artifact/ArtifactStagingServiceTest.java
@@ -43,6 +43,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -56,6 +57,7 @@ public class ArtifactStagingServiceTest {
   private Path stagingDir;
   @Rule public TemporaryFolder tempFolder = new TemporaryFolder();
   @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Before
   public void setUp() throws Exception {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/BundleFinalizationHandlersTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/BundleFinalizationHandlersTest.java
index 9494104b4c68..ea2f00e7e23b 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/BundleFinalizationHandlersTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/BundleFinalizationHandlersTest.java
@@ -24,13 +24,17 @@
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.FinalizeBundleRequest;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest;
 import org.apache.beam.runners.fnexecution.control.BundleFinalizationHandlers.InMemoryFinalizer;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link BundleFinalizationHandlers}. */
 @RunWith(JUnit4.class)
 public class BundleFinalizationHandlersTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
+
   @Test
   public void testInMemoryFinalizer() {
     InstructionRequestHandler mockHandler = mock(InstructionRequestHandler.class);
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java
index 7adc78a22ef8..823af895d655 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/DefaultJobBundleFactoryTest.java
@@ -77,6 +77,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.InOrder;
@@ -88,6 +89,7 @@
 @RunWith(JUnit4.class)
 public class DefaultJobBundleFactoryTest {
   @Rule public ExpectedException thrown = ExpectedException.none();
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Mock private EnvironmentFactory envFactory;
   @Mock private RemoteEnvironment remoteEnvironment;
   @Mock private InstructionRequestHandler instructionHandler;
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java
index b36cac381ac2..b605ae407e19 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientPoolServiceTest.java
@@ -48,13 +48,16 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Unit tests for {@link FnApiControlClientPoolService}. */
 @RunWith(JUnit4.class)
 public class FnApiControlClientPoolServiceTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final String WORKER_ID = "test_worker_id";
   private final ControlClientPool pool = MapControlClientPool.create();
   private final FnApiControlClientPoolService controlService =
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java
index 30cd81ec5ba7..293f847bcc18 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/FnApiControlClientTest.java
@@ -38,6 +38,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -47,7 +48,7 @@
 /** Unit tests for {@link FnApiControlClient}. */
 @RunWith(JUnit4.class)
 public class FnApiControlClientTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Rule public ExpectedException thrown = ExpectedException.none();
 
   @Mock public StreamObserver<BeamFnApi.InstructionRequest> mockObserver;
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java
index 47df7b7c9735..108412a9dafb 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ProcessBundleDescriptorsTest.java
@@ -60,13 +60,16 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 
 /** Tests for {@link ProcessBundleDescriptors}. */
 // TODO(https://github.com/apache/beam/issues/21230): Remove when new version of errorprone is
 // released (2.11.0)
 @SuppressWarnings("unused")
 public class ProcessBundleDescriptorsTest implements Serializable {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   /**
    * Tests that a stateful stage will wrap the key coder of a stateful transform in a
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactoryTest.java
index be28ae06acc0..a7ba09f8fcec 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/ReferenceCountingExecutableStageContextFactoryTest.java
@@ -28,12 +28,14 @@
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link ReferenceCountingExecutableStageContextFactory}. */
 @RunWith(JUnit4.class)
 public class ReferenceCountingExecutableStageContextFactoryTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Rule
   public ExpectedLogs expectedLogs =
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java
index ea23e28ddb66..d827421d2e81 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/RemoteExecutionTest.java
@@ -157,6 +157,7 @@
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.slf4j.LoggerFactory;
@@ -174,7 +175,7 @@
   "unused"
 })
 public class RemoteExecutionTest implements Serializable {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Rule public transient ResetDateTimeProvider resetDateTimeProvider = new ResetDateTimeProvider();
 
   private static final String WORKER_ID = "remote_test";
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java
index 6ebe5eca3e65..2168854c863d 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SdkHarnessClientTest.java
@@ -95,6 +95,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.ArgumentCaptor;
@@ -110,7 +111,7 @@
   "rawtypes", // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class SdkHarnessClientTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Mock public FnApiControlClient fnApiControlClient;
   @Mock public FnDataService dataService;
   @Captor ArgumentCaptor<CloseableFnDataReceiver<BeamFnApi.Elements>> outputReceiverCaptor;
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java
index 11973385fe16..61090e9ffdf3 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/SingleEnvironmentInstanceJobBundleFactoryTest.java
@@ -51,7 +51,9 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -60,6 +62,7 @@
 /** Tests for {@link SingleEnvironmentInstanceJobBundleFactory}. */
 @RunWith(JUnit4.class)
 public class SingleEnvironmentInstanceJobBundleFactoryTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Mock private EnvironmentFactory environmentFactory;
   @Mock private InstructionRequestHandler instructionRequestHandler;
 
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactoryTest.java
index 2d6ff53b78aa..14bbe751d3c5 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/control/TimerReceiverFactoryTest.java
@@ -20,12 +20,16 @@
 import static org.junit.Assert.assertEquals;
 
 import org.apache.beam.sdk.values.KV;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 @RunWith(JUnit4.class)
 public class TimerReceiverFactoryTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
+
   @Test
   public void testEncodeAndDecode() {
     KV<String, String> expected = KV.of("123:\"ab\n'c:", "456:sdf:d");
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/GrpcDataServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/GrpcDataServiceTest.java
index b04ef8881070..b06c38db7e74 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/GrpcDataServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/GrpcDataServiceTest.java
@@ -54,13 +54,16 @@
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link GrpcDataService}. */
 @RunWith(JUnit4.class)
 public class GrpcDataServiceTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final String TRANSFORM_ID = "888";
   private static final Coder<WindowedValue<String>> CODER =
       LengthPrefixCoder.of(WindowedValue.getValueOnlyCoder(StringUtf8Coder.of()));
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/RemoteInputDestinationTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/RemoteInputDestinationTest.java
index 9b2f919e8a05..40f48d012c31 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/RemoteInputDestinationTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/data/RemoteInputDestinationTest.java
@@ -26,13 +26,16 @@
 import org.apache.beam.sdk.coders.LengthPrefixCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
 import org.apache.beam.sdk.values.KV;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link RemoteInputDestination}. */
 @RunWith(JUnit4.class)
 public class RemoteInputDestinationTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Test
   public void testConstruction() {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java
index e7d294fffc5d..7d5af0514dac 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerCommandTest.java
@@ -33,6 +33,7 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
@@ -40,7 +41,7 @@
 @Category(NeedsDocker.class)
 @RunWith(JUnit4.class)
 public class DockerCommandTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Rule public ExpectedException thrown = ExpectedException.none();
 
   @Test
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactoryTest.java
index 4b2948ecf9f8..16d32e46f010 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/DockerEnvironmentFactoryTest.java
@@ -48,6 +48,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameter;
@@ -58,7 +59,7 @@
 
 /** Tests for {@link DockerEnvironmentFactory}. */
 public class DockerEnvironmentFactoryTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final ApiServiceDescriptor SERVICE_DESCRIPTOR =
       ApiServiceDescriptor.newBuilder().setUrl("service-url").build();
   private static final String IMAGE_NAME = "my-image";
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactoryTest.java
index efddc7226089..4ba993251739 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentFactoryTest.java
@@ -40,7 +40,9 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.RemoteEnvironmentOptions;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -50,7 +52,7 @@
 /** Tests for {@link ProcessEnvironmentFactory}. */
 @RunWith(JUnit4.class)
 public class ProcessEnvironmentFactoryTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final ApiServiceDescriptor SERVICE_DESCRIPTOR =
       ApiServiceDescriptor.newBuilder().setUrl("service-url").build();
   private static final String COMMAND = "my-command";
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentTest.java
index aefcc2954a12..2b35d81a5300 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessEnvironmentTest.java
@@ -22,13 +22,16 @@
 
 import org.apache.beam.model.pipeline.v1.RunnerApi.Environment;
 import org.apache.beam.runners.fnexecution.control.InstructionRequestHandler;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link ProcessEnvironment}. */
 @RunWith(JUnit4.class)
 public class ProcessEnvironmentTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Test
   public void closeClosesInstructionRequestHandler() throws Exception {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessManagerTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessManagerTest.java
index 8108cd4349e4..4074bf949439 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessManagerTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/ProcessManagerTest.java
@@ -34,13 +34,16 @@
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.Collections;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link ProcessManager}. */
 @RunWith(JUnit4.class)
 public class ProcessManagerTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Test
   public void testRunSimpleCommand() throws IOException {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/RemoteEnvironmentTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/RemoteEnvironmentTest.java
index af8a810cef77..9bb9eb616117 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/RemoteEnvironmentTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/environment/RemoteEnvironmentTest.java
@@ -25,13 +25,16 @@
 import org.apache.beam.model.pipeline.v1.RunnerApi.Environment;
 import org.apache.beam.runners.core.construction.Environments;
 import org.apache.beam.runners.fnexecution.control.InstructionRequestHandler;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link RemoteEnvironment}. */
 @RunWith(JUnit4.class)
 public class RemoteEnvironmentTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Test
   public void closeClosesInstructionRequestHandler() throws Exception {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingServiceTest.java
index 7a29d02fd355..2fcbde5eaa31 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/logging/GrpcLoggingServiceTest.java
@@ -41,13 +41,16 @@
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.ManagedChannel;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link GrpcLoggingService}. */
 @RunWith(JUnit4.class)
 public class GrpcLoggingServiceTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private Consumer<LogControl> messageDiscarder =
       item -> {
         // Ignore
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/provisioning/StaticGrpcProvisionServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/provisioning/StaticGrpcProvisionServiceTest.java
index bef964bf186b..f0d37fe34d3c 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/provisioning/StaticGrpcProvisionServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/provisioning/StaticGrpcProvisionServiceTest.java
@@ -34,13 +34,17 @@
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Struct;
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.Value;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.inprocess.InProcessChannelBuilder;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Tests for {@link StaticGrpcProvisionService}. */
 @RunWith(JUnit4.class)
 public class StaticGrpcProvisionServiceTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
+
   @Test
   public void returnsProvisionInfo() throws Exception {
     Struct options =
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/GrpcStateServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/GrpcStateServiceTest.java
index 0342ddd67a55..fffc0321ca2c 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/GrpcStateServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/GrpcStateServiceTest.java
@@ -34,7 +34,9 @@
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -47,6 +49,7 @@
   "rawtypes", // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class GrpcStateServiceTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final long TIMEOUT_MS = 30 * 1000;
 
   private GrpcStateService stateService;
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java
index 5529133b5b49..3ef70756760a 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/state/StateRequestHandlersTest.java
@@ -39,7 +39,9 @@
 import org.apache.beam.sdk.util.CoderUtils;
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mockito;
@@ -50,6 +52,8 @@
   "rawtypes" // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class StateRequestHandlersTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
+
   @Test
   public void testDelegatingStateHandlerDelegates() throws Exception {
     StateRequestHandler mockHandler = Mockito.mock(StateRequestHandler.class);
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java
index e1d6401d5e53..49ced362c29a 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/BeamWorkerStatusGrpcServiceTest.java
@@ -48,6 +48,7 @@
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -55,6 +56,7 @@
 
 @RunWith(JUnit4.class)
 public class BeamWorkerStatusGrpcServiceTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Rule public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
   private static final String ID = "id";
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/WorkerStatusClientTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/WorkerStatusClientTest.java
index 49e621fb85fe..0a250ef0b8d3 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/WorkerStatusClientTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/status/WorkerStatusClientTest.java
@@ -29,7 +29,9 @@
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -37,7 +39,7 @@
 
 @RunWith(JUnit4.class)
 public class WorkerStatusClientTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   @Mock public StreamObserver<BeamFnApi.WorkerStatusRequest> mockObserver;
   private WorkerStatusClient client;
 
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactoryTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactoryTest.java
index 1f459456d13b..a5f23db5b29e 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactoryTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/BatchSideInputHandlerFactoryTest.java
@@ -55,6 +55,7 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 import org.mockito.Mock;
@@ -77,6 +78,7 @@ public class BatchSideInputHandlerFactoryTest {
                       COLLECTION_ID, RunnerApi.PCollection.getDefaultInstance()))));
 
   @Rule public ExpectedException thrown = ExpectedException.none();
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Mock private BatchSideInputHandlerFactory.SideInputGetter context;
 
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java
index a0fbd26d72f1..40e8ab96c615 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/translation/PipelineTranslatorUtilsTest.java
@@ -25,10 +25,13 @@
 import java.util.Map;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.BiMap;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 
 /** Tests for {@link PipelineTranslatorUtils}. */
 public class PipelineTranslatorUtilsTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   @Test
   public void testOutputMapCreation() {
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java
index 4442674f4e6e..28144e68118a 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/ByteStringCoderTest.java
@@ -36,13 +36,14 @@
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
 
 /** Test case for {@link ByteStringCoder}. */
 @RunWith(JUnit4.class)
 public class ByteStringCoderTest {
-
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final ByteStringCoder TEST_CODER = ByteStringCoder.of();
 
   private static final List<String> TEST_STRING_VALUES =
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java
index 389cdff232a2..5a775c2c98ec 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/CommonCoderTest.java
@@ -99,7 +99,9 @@
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameter;
@@ -111,6 +113,7 @@
   "rawtypes", // TODO(https://github.com/apache/beam/issues/20447)
 })
 public class CommonCoderTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
   private static final String STANDARD_CODERS_YAML_PATH =
       "/org/apache/beam/model/fnexecution/v1/standard_coders.yaml";
 
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java
index ddce63a19c99..86917e1c6bd4 100644
--- a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/wire/LengthPrefixUnknownCodersTest.java
@@ -37,7 +37,9 @@
 import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.Timeout;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameter;
@@ -46,6 +48,7 @@
 /** Tests for {@link LengthPrefixUnknownCoders}. */
 @RunWith(Parameterized.class)
 public class LengthPrefixUnknownCodersTest {
+  @Rule public transient Timeout globalTimeout = Timeout.seconds(600);
 
   private static class UnknownCoder extends CustomCoder<String> {
     private static final Coder<?> INSTANCE = new UnknownCoder();
diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java
index 17efbf9a06ec..41e6135b9207 100644
--- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java
+++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/InMemoryJobService.java
@@ -49,6 +49,7 @@
 import org.apache.beam.model.jobmanagement.v1.JobServiceGrpc;
 import org.apache.beam.model.pipeline.v1.Endpoints;
 import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.runners.core.construction.Environments;
 import org.apache.beam.runners.core.construction.graph.PipelineValidator;
 import org.apache.beam.runners.fnexecution.artifact.ArtifactStagingService;
 import org.apache.beam.sdk.fn.server.FnService;
@@ -62,7 +63,6 @@
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.StatusRuntimeException;
 import org.apache.beam.vendor.grpc.v1p54p0.io.grpc.stub.StreamObserver;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions;
-import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Maps;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -198,11 +198,7 @@ public void prepare(
 
       stagingService
           .getService()
-          .registerJob(
-              stagingSessionToken,
-              Maps.transformValues(
-                  request.getPipeline().getComponents().getEnvironmentsMap(),
-                  RunnerApi.Environment::getDependenciesList));
+          .registerJob(stagingSessionToken, extractDependencies(request.getPipeline()));
 
       // send response
       PrepareJobResponse response =
@@ -287,26 +283,50 @@ public void run(RunJobRequest request, StreamObserver<RunJobResponse> responseOb
     }
   }
 
+  private Map<String, List<RunnerApi.ArtifactInformation>> extractDependencies(
+      RunnerApi.Pipeline pipeline) {
+    Map<String, List<RunnerApi.ArtifactInformation>> dependencies = new HashMap<>();
+    for (Map.Entry<String, RunnerApi.Environment> entry :
+        pipeline.getComponents().getEnvironmentsMap().entrySet()) {
+      List<RunnerApi.Environment> subEnvs = Environments.expandAnyOfEnvironments(entry.getValue());
+      for (int i = 0; i < subEnvs.size(); i++) {
+        dependencies.put(i + ":" + entry.getKey(), subEnvs.get(i).getDependenciesList());
+      }
+    }
+    return dependencies;
+  }
+
   private RunnerApi.Pipeline resolveDependencies(RunnerApi.Pipeline pipeline, String stagingToken) {
     Map<String, List<RunnerApi.ArtifactInformation>> resolvedDependencies =
         stagingService.getService().getStagedArtifacts(stagingToken);
     Map<String, RunnerApi.Environment> newEnvironments = new HashMap<>();
     for (Map.Entry<String, RunnerApi.Environment> entry :
         pipeline.getComponents().getEnvironmentsMap().entrySet()) {
-      if (entry.getValue().getDependenciesCount() > 0 && resolvedDependencies == null) {
-        throw new RuntimeException(
-            "Artifact dependencies provided but not staged for " + entry.getKey());
+      List<RunnerApi.Environment> subEnvs = Environments.expandAnyOfEnvironments(entry.getValue());
+      List<RunnerApi.Environment> newSubEnvs = new ArrayList<>();
+      for (int i = 0; i < subEnvs.size(); i++) {
+        RunnerApi.Environment subEnv = subEnvs.get(i);
+        if (subEnv.getDependenciesCount() > 0 && resolvedDependencies == null) {
+          throw new RuntimeException(
+              "Artifact dependencies provided but not staged for " + entry.getKey());
+        }
+        newSubEnvs.add(
+            subEnv.getDependenciesCount() == 0
+                ? subEnv
+                : subEnv
+                    .toBuilder()
+                    .clearDependencies()
+                    .addAllDependencies(resolvedDependencies.get(i + ":" + entry.getKey()))
+                    .build());
+      }
+      if (newSubEnvs.size() == 1) {
+        newEnvironments.put(entry.getKey(), newSubEnvs.get(0));
+      } else {
+        newEnvironments.put(
+            entry.getKey(),
+            Environments.createAnyOfEnvironment(
+                newSubEnvs.toArray(new RunnerApi.Environment[newSubEnvs.size()])));
       }
-      newEnvironments.put(
-          entry.getKey(),
-          entry.getValue().getDependenciesCount() == 0
-              ? entry.getValue()
-              : entry
-                  .getValue()
-                  .toBuilder()
-                  .clearDependencies()
-                  .addAllDependencies(resolvedDependencies.get(entry.getKey()))
-                  .build());
     }
     RunnerApi.Pipeline.Builder builder = pipeline.toBuilder();
     builder.getComponentsBuilder().clearEnvironments().putAllEnvironments(newEnvironments);
diff --git a/runners/portability/java/build.gradle b/runners/portability/java/build.gradle
index ee6fe3b8fd8d..69b93e9d3676 100644
--- a/runners/portability/java/build.gradle
+++ b/runners/portability/java/build.gradle
@@ -200,13 +200,6 @@ def createUlrValidatesRunnerTask = { name, environmentType, dockerImageTask = ""
       // https://github.com/apache/beam/issues/20374
       excludeTestsMatching 'org.apache.beam.sdk.transforms.CombineTest$BasicTests.testHotKeyCombiningWithAccumulationMode'
 
-      // https://github.com/apache/beam/issues/20878
-      excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoTest$MultipleInputsAndOutputTests.testSideInputAnnotationWithMultipleSideInputs'
-      excludeTestsMatching 'org.apache.beam.sdk.transforms.ViewTest.testMapAsEntrySetSideInput'
-      excludeTestsMatching 'org.apache.beam.sdk.transforms.ViewTest.testWindowedMultimapAsEntrySetSideInput'
-      excludeTestsMatching 'org.apache.beam.sdk.transforms.ViewTest.testWindowedMapAsEntrySetSideInput'
-      excludeTestsMatching 'org.apache.beam.sdk.transforms.ViewTest.testMultimapAsEntrySetSideInput'
-
       for (String test : sickbayTests) {
         excludeTestsMatching test
       }
diff --git a/runners/samza/src/test/resources/ExpectedDag.json b/runners/samza/src/test/resources/ExpectedDag.json
index 3165fc84958c..c61b80134d8a 100644
--- a/runners/samza/src/test/resources/ExpectedDag.json
+++ b/runners/samza/src/test/resources/ExpectedDag.json
@@ -98,26 +98,26 @@
                                   "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements",
                                   "ChildNodes": [
                                     {
-                                      "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/Window.Into()",
+                                      "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/SetIdentityWindow",
                                       "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle",
                                       "ChildNodes": [
                                         {
-                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/Window.Into()/Window.Assign",
-                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/Window.Into()"
+                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/SetIdentityWindow/Window.Assign",
+                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/SetIdentityWindow"
                                         }
                                       ]
                                     },
                                     {
-                                      "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps",
+                                      "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata",
                                       "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle",
                                       "ChildNodes": [
                                         {
-                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps/ParDo(Anonymous)",
-                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps",
+                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata/ParDo(Anonymous)",
+                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata",
                                           "ChildNodes": [
                                             {
-                                              "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps/ParDo(Anonymous)/ParMultiDo(Anonymous)",
-                                              "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps/ParDo(Anonymous)"
+                                              "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous)",
+                                              "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata/ParDo(Anonymous)"
                                             }
                                           ]
                                         }
@@ -138,38 +138,16 @@
                                       ]
                                     },
                                     {
-                                      "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps",
+                                      "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata",
                                       "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle",
                                       "ChildNodes": [
                                         {
-                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard",
-                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps",
+                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)",
+                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata",
                                           "ChildNodes": [
                                             {
-                                              "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard/ParDo(Anonymous)",
-                                              "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard",
-                                              "ChildNodes": [
-                                                {
-                                                  "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard/ParDo(Anonymous)/ParMultiDo(Anonymous)",
-                                                  "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard/ParDo(Anonymous)"
-                                                }
-                                              ]
-                                            }
-                                          ]
-                                        },
-                                        {
-                                          "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues",
-                                          "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps",
-                                          "ChildNodes": [
-                                            {
-                                              "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)",
-                                              "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues",
-                                              "ChildNodes": [
-                                                {
-                                                  "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)/ParMultiDo(Anonymous)",
-                                                  "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)"
-                                                }
-                                              ]
+                                              "fullName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous)",
+                                              "enclosingNode": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)"
                                             }
                                           ]
                                         }
@@ -287,14 +265,14 @@
     },
     {
       "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/Assign unique key/AddKeys/Map/ParMultiDo(Anonymous)",
-      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/Window.Into()/Window.Assign"
+      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/SetIdentityWindow/Window.Assign"
     },
     {
-      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/Window.Into()/Window.Assign",
-      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps/ParDo(Anonymous)/ParMultiDo(Anonymous)"
+      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/SetIdentityWindow/Window.Assign",
+      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous)"
     },
     {
-      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalTimestamps/ParDo(Anonymous)/ParMultiDo(Anonymous)",
+      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ReifyOriginalMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous)",
       "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/GroupByKey"
     },
     {
@@ -303,14 +281,10 @@
     },
     {
       "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/ExpandIterable/ParMultiDo(Anonymous)",
-      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard/ParDo(Anonymous)/ParMultiDo(Anonymous)"
-    },
-    {
-      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/ReifyTimestamps.RemoveWildcard/ParDo(Anonymous)/ParMultiDo(Anonymous)",
-      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)/ParMultiDo(Anonymous)"
+      "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous)"
     },
     {
-      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)/ParMultiDo(Anonymous)",
+      "from": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous)",
       "to": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Drop key/Values/Map/ParMultiDo(Anonymous)"
     },
     {
@@ -357,7 +331,7 @@
     },
     {
       "transformName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Drop key/Values/Map/ParMultiDo(Anonymous)",
-      "inputs": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)/ParMultiDo(Anonymous).output",
+      "inputs": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous).output",
       "outputs": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Drop key/Values/Map/ParMultiDo(Anonymous).output"
     },
     {
@@ -378,7 +352,7 @@
     {
       "transformName": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle",
       "inputs": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/Assign unique key/AddKeys/Map/ParMultiDo(Anonymous).output",
-      "outputs": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreOriginalTimestamps/Reify.ExtractTimestampsFromValues/ParDo(Anonymous)/ParMultiDo(Anonymous).output"
+      "outputs": "Create.TimestampedValues/Create.Values/Read(CreateSource)/ParDo(BoundedSourceAsSDFWrapper)/ParMultiDo(BoundedSourceAsSDFWrapper)/ProcessKeyedElements/Reshuffle/RestoreMetadata/ParDo(Anonymous)/ParMultiDo(Anonymous).output"
     },
     {
       "transformName": "Create.TimestampedValues/ParDo(ConvertTimestamps)/ParMultiDo(ConvertTimestamps)",
diff --git a/sdks/go.mod b/sdks/go.mod
index d64ee5033383..1c5874a49c1e 100644
--- a/sdks/go.mod
+++ b/sdks/go.mod
@@ -24,28 +24,28 @@ go 1.20
 
 require (
 	cloud.google.com/go/bigquery v1.57.1
-	cloud.google.com/go/bigtable v1.20.0
+	cloud.google.com/go/bigtable v1.21.0
 	cloud.google.com/go/datastore v1.15.0
 	cloud.google.com/go/profiler v0.4.0
 	cloud.google.com/go/pubsub v1.33.0
-	cloud.google.com/go/spanner v1.51.0
-	cloud.google.com/go/storage v1.35.1
-	github.com/aws/aws-sdk-go-v2 v1.23.2
-	github.com/aws/aws-sdk-go-v2/config v1.25.8
-	github.com/aws/aws-sdk-go-v2/credentials v1.16.6
+	cloud.google.com/go/spanner v1.53.1
+	cloud.google.com/go/storage v1.36.0
+	github.com/aws/aws-sdk-go-v2 v1.24.0
+	github.com/aws/aws-sdk-go-v2/config v1.26.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.13
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2
-	github.com/aws/smithy-go v1.17.0
+	github.com/aws/smithy-go v1.19.0
 	github.com/docker/go-connections v0.4.0
 	github.com/dustin/go-humanize v1.0.1
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/golang/protobuf v1.5.3 // TODO(danoliveira): Fully replace this with google.golang.org/protobuf
 	github.com/google/go-cmp v0.6.0
-	github.com/google/uuid v1.4.0
+	github.com/google/uuid v1.5.0
 	github.com/johannesboyne/gofakes3 v0.0.0-20221110173912-32fb85c5aed6
 	github.com/lib/pq v1.10.9
 	github.com/linkedin/goavro/v2 v2.12.0
-	github.com/nats-io/nats-server/v2 v2.10.5
+	github.com/nats-io/nats-server/v2 v2.10.7
 	github.com/nats-io/nats.go v1.31.0
 	github.com/proullon/ramsql v0.1.3
 	github.com/spf13/cobra v1.8.0
@@ -53,15 +53,15 @@ require (
 	github.com/tetratelabs/wazero v1.5.0
 	github.com/xitongsys/parquet-go v1.6.2
 	github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c
-	go.mongodb.org/mongo-driver v1.12.1
-	golang.org/x/net v0.18.0
-	golang.org/x/oauth2 v0.13.0
+	go.mongodb.org/mongo-driver v1.13.1
+	golang.org/x/net v0.19.0
+	golang.org/x/oauth2 v0.15.0
 	golang.org/x/sync v0.5.0
-	golang.org/x/sys v0.14.0
+	golang.org/x/sys v0.15.0
 	golang.org/x/text v0.14.0
-	google.golang.org/api v0.151.0
-	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b
-	google.golang.org/grpc v1.59.0
+	google.golang.org/api v0.154.0
+	google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f
+	google.golang.org/grpc v1.60.1
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/retry.v1 v1.0.3
 	gopkg.in/yaml.v2 v2.4.0
@@ -69,7 +69,7 @@ require (
 )
 
 require (
-	github.com/fsouza/fake-gcs-server v1.47.6
+	github.com/fsouza/fake-gcs-server v1.47.7
 	golang.org/x/exp v0.0.0-20230807204917-050eac23e9de
 )
 
@@ -77,6 +77,8 @@ require (
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Microsoft/hcsshim v0.11.1 // indirect
 	github.com/containerd/log v0.1.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/minio/highwayhash v1.0.2 // indirect
@@ -89,15 +91,20 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	golang.org/x/time v0.4.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
+	go.opentelemetry.io/otel v1.21.0 // indirect
+	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/trace v1.21.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
 )
 
 require (
-	cloud.google.com/go v0.110.8 // indirect
-	cloud.google.com/go/compute v1.23.1 // indirect
+	cloud.google.com/go v0.110.10 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.3 // indirect
-	cloud.google.com/go/longrunning v0.5.2 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
+	cloud.google.com/go/longrunning v0.5.4 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -106,18 +113,18 @@ require (
 	github.com/apache/thrift v0.16.0 // indirect
 	github.com/aws/aws-sdk-go v1.34.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.17.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.25.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -130,7 +137,7 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.11.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
-	github.com/felixge/httpsnoop v1.0.2 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/goccy/go-json v0.9.11 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -141,13 +148,13 @@ require (
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
-	github.com/gorilla/handlers v1.5.1 // indirect
-	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/gorilla/handlers v1.5.2 // indirect
+	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
+	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
@@ -173,11 +180,11 @@ require (
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.15.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
 )
diff --git a/sdks/go.sum b/sdks/go.sum
index 45f8971db12e..7dc312f9cba2 100644
--- a/sdks/go.sum
+++ b/sdks/go.sum
@@ -8,29 +8,29 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
 cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME=
-cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk=
+cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y=
+cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.57.1 h1:FiULdbbzUxWD0Y4ZGPSVCDLvqRSyCIO6zKV7E2nf5uA=
 cloud.google.com/go/bigquery v1.57.1/go.mod h1:iYzC0tGVWt1jqSzBHqCr3lrRn0u13E8e+AqowBsDgug=
-cloud.google.com/go/bigtable v1.20.0 h1:NqZC/WcesSn4O8L0I2JmuNsUigSyBQifVLYgM9LMQeQ=
-cloud.google.com/go/bigtable v1.20.0/go.mod h1:upJDn8frsjzpRMfybiWkD1PG6WCCL7CRl26MgVeoXY4=
-cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0=
-cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78=
+cloud.google.com/go/bigtable v1.21.0 h1:BFN4jhkA9ULYYV2Ug7AeOtetVLnN2jKuIq5TcRc5C38=
+cloud.google.com/go/bigtable v1.21.0/go.mod h1:V0sYNRtk0dgAKjyRr/MyBpHpSXqh+9P39euf820EZ74=
+cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
+cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/datacatalog v1.18.1 h1:xJp9mZrc2HPaoxIz3sP9pCmf/impifweQ/yGG9VBfio=
+cloud.google.com/go/datacatalog v1.19.0 h1:rbYNmHwvAOOwnW2FPXYkaK3Mf1MmGqRzK0mMiIEyLdo=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/datastore v1.15.0 h1:0P9WcsQeTWjuD1H14JIY7XQscIPQ4Laje8ti96IC5vg=
 cloud.google.com/go/datastore v1.15.0/go.mod h1:GAeStMBIt9bPS7jMJA85kgkpsMkvseWWXiaHya9Jes8=
-cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc=
-cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE=
-cloud.google.com/go/kms v1.15.3 h1:RYsbxTRmk91ydKCzekI2YjryO4c5Y2M80Zwcs9/D/cI=
-cloud.google.com/go/longrunning v0.5.2 h1:u+oFqfEwwU7F9dIELigxbe0XVnBAo9wqMuQLA50CZ5k=
-cloud.google.com/go/longrunning v0.5.2/go.mod h1:nqo6DQbNV2pXhGDbDMoN2bWz68MjZUzqv2YttZiveCs=
+cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
+cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
+cloud.google.com/go/kms v1.15.5 h1:pj1sRfut2eRbD9pFRjNnPNg/CzJPuQAzUujMIM1vVeM=
+cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg=
+cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI=
 cloud.google.com/go/profiler v0.4.0 h1:ZeRDZbsOBDyRG0OiK0Op1/XWZ3xeLwJc9zjkzczUxyY=
 cloud.google.com/go/profiler v0.4.0/go.mod h1:RvPlm4dilIr3oJtAOeFQU9Lrt5RoySHSDj4pTd6TWeU=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
@@ -38,13 +38,13 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g=
 cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc=
-cloud.google.com/go/spanner v1.51.0 h1:l3exhhsVMKsx1E7Xd1QajYSvHmI1KZoWPW5tRxIIdvQ=
-cloud.google.com/go/spanner v1.51.0/go.mod h1:c5KNo5LQ1X5tJwma9rSQZsXNBDNvj4/n8BVc3LNahq0=
+cloud.google.com/go/spanner v1.53.1 h1:xNmE0SXMSxNBuk7lRZ5G/S+A49X91zkSTt7Jn5Ptlvw=
+cloud.google.com/go/spanner v1.53.1/go.mod h1:liG4iCeLqm5L3fFLU5whFITqP0e0orsAW1uUSrd4rws=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.35.1 h1:B59ahL//eDfx2IIKFBeT5Atm9wnNmj3+8xG/W4WB//w=
-cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
+cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8=
+cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -81,39 +81,39 @@ github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve
 github.com/aws/aws-sdk-go v1.34.0 h1:brux2dRrlwCF5JhTL7MUT3WUwo9zfDHZZp3+g3Mvlmo=
 github.com/aws/aws-sdk-go v1.34.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250=
-github.com/aws/aws-sdk-go-v2 v1.23.2 h1:UoTll1Y5b88x8h53OlsJGgOHwpggdMr7UVnLjMb3XYg=
-github.com/aws/aws-sdk-go-v2 v1.23.2/go.mod h1:i1XDttT4rnf6vxc9AuskLc6s7XBee8rlLilKlc03uAA=
+github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk=
+github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 h1:ZY3108YtBNq96jNZTICHxN1gSBSbnvIdYwwqnvCV4Mc=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1/go.mod h1:t8PYl/6LzdAqsU4/9tz28V/kU+asFePvpOMkdul0gEQ=
 github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA=
-github.com/aws/aws-sdk-go-v2/config v1.25.8 h1:CHr7PIzyfevjNiqL9rU6xoqHZKCO2ldY6LmvRDfpRuI=
-github.com/aws/aws-sdk-go-v2/config v1.25.8/go.mod h1:zefIy117FDPOVU0xSOFG8mx9kJunuVopzI639tjYXc0=
+github.com/aws/aws-sdk-go-v2/config v1.26.2 h1:+RWLEIWQIGgrz2pBPAUoGgNGs1TOyF4Hml7hCnYj2jc=
+github.com/aws/aws-sdk-go-v2/config v1.26.2/go.mod h1:l6xqvUxt0Oj7PI/SUXYLNyZ9T/yBPn3YTQcJLLOdtR8=
 github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.6 h1:TimIpn1p4v44i0sJMKsnpby1P9sP1ByKLsdm7bvOmwM=
-github.com/aws/aws-sdk-go-v2/credentials v1.16.6/go.mod h1:+CLPlYf9FQLeXD8etOYiZxpLQqc3GL4EikxjkFFp1KA=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13 h1:WLABQ4Cp4vXtXfOWOS3MEZKr6AAYUpMczLhgKtAjQ/8=
+github.com/aws/aws-sdk-go-v2/credentials v1.16.13/go.mod h1:Qg6x82FXwW0sJHzYruxGiuApNo31UEtJvXVSZAXeWiw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.6 h1:pPs23/JLSOlwnmSRNkdbt3upmBeF6QL/3MHEb6KzTyo=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.6/go.mod h1:jsoDHV44SxWv00wlbx0yA5M7n5rmE5rGk+OGA0suXSw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8 h1:wuOjvalpd2CnXffks74Vq6n3yv9vunKCoy4R1sjStGk=
 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.13.8/go.mod h1:vywwjy6VnrR48Izg136JoSUXC4mH9QeUi3g0EH9DSrA=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.5 h1:16Z1XuMUv63fcyW5bIUno6AFcX4drsrE0gof+xue6g4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.5/go.mod h1:pRvFacV2qbRKy34ZFptHZW4wpauJA445bqFbvA6ikSo=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.5 h1:RxpMuBgzP3Dj1n5CZY6droLFcsn5gc7QsrIcaGQoeCs=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.5/go.mod h1:dO8Js7ym4Jzg/wcjTgCRVln/jFn3nI82XNhsG2lWbDI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3 h1:lMwCXiWJlrtZot0NJTjbC8G9zl+V3i68gBTBBvDeEXA=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.3/go.mod h1:5yzAuE9i2RkVAttBl8yxZgQr5OCq4D5yDnG7j9x2L0U=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1/go.mod h1:v33JQ57i2nekYTA70Mb+O18KeH4KqhdqxTJZNK1zdRE=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1 h1:rpkF4n0CyFcrJUG/rNNohoTmhtWlFTRI4BsZOh9PvLs=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.1/go.mod h1:l9ymW25HOqymeU2m1gbUQ3rUIsTwKs8gYHXkqDQUhiI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3 h1:xbwRyCy7kXrOj89iIKLB6NfE2WCpP9HoKyk8dMDvnIQ=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.3/go.mod h1:R+/S1O4TYpcktbVwddeOYg+uwUfLhADP2S/x4QwsCTM=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.5 h1:F+XafeiK7Uf4YwTZfe/JLt+3cB6je9sI7l0TY4f2CkY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.5/go.mod h1:NlZuvlkyu6l/F3+qIBsGGtYLL2Z71tCf5NFoNAaG1NY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1/go.mod h1:6EQZIwNNvHpq/2/QSJnp4+ECvqIy55w95Ofs0ze+nGQ=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3 h1:KV0z2RDc7euMtg8aUT1czv5p29zcLlXALNFsd3jkkEc=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.3/go.mod h1:KZgs2ny8HsxRIRbDwgvJcHHBZPOzQr/+NtGwnP+w2ec=
@@ -121,16 +121,16 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32
 github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2 h1:NnduxUd9+Fq9DcCDdJK8v6l9lR1xDX4usvog+JuQAno=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.42.2/go.mod h1:NXRKkiRF+erX2hnybnVU660cYT5/KChRD4iUgJ97cI8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.17.5 h1:kuK22ZsITfzaZEkxEl5H/lhy2k3G4clBtcQBI93RbIc=
-github.com/aws/aws-sdk-go-v2/service/sso v1.17.5/go.mod h1:/tLqstwPfJLHYGBB5/c8P1ITI82pcGs7cJQuXku2pOg=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.3 h1:l5d5nrTFMhiUWNoLnV7QNI4m42/3WVSXqSyqVy+elGk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.20.3/go.mod h1:30gKZp2pHQJq3yTmVy+hJKDFynSoYzVqYaxe4yPi+xI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38=
 github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg=
-github.com/aws/aws-sdk-go-v2/service/sts v1.25.6 h1:39dJNBt35p8dFSnQdoy+QbDaPenTxFqqDQFOb1GDYpE=
-github.com/aws/aws-sdk-go-v2/service/sts v1.25.6/go.mod h1:6DKEi+8OnUrqEEh6OCam16AYQHWAOyNgRiUGnHoh7Cg=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 h1:HJeiuZ2fldpd0WqngyMR6KW7ofkXNLyOaHwEIGm39Cs=
+github.com/aws/aws-sdk-go-v2/service/sts v1.26.6/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU=
 github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
-github.com/aws/smithy-go v1.17.0 h1:wWJD7LX6PBV6etBUwO0zElG0nWN9rUhp0WdYeHSHAaI=
-github.com/aws/smithy-go v1.17.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
+github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM=
+github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -186,19 +186,23 @@ github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQ
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
 github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
-github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
-github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20=
 github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/fsouza/fake-gcs-server v1.47.6 h1:/d/879q/Os9Zc5gyV3QVLfZoajN1KcWucf2zYCFeFxs=
-github.com/fsouza/fake-gcs-server v1.47.6/go.mod h1:ApSXKexpG1BUXJ4f2tNCxvhTKwCPFqFLBDW2UNQDODE=
+github.com/fsouza/fake-gcs-server v1.47.7 h1:56/U4rKY081TaNbq0gHWi7/71UxC2KROqcnrD9BRJhs=
+github.com/fsouza/fake-gcs-server v1.47.7/go.mod h1:4vPUynN8/zZlxk5Jpy6LvvTTxItdTAObK4DYnp89Jys=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gorp/gorp v2.2.0+incompatible h1:xAUh4QgEeqPPhK3vxZN+bzrim1z5Av6q837gtjUlshc=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -236,6 +240,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -279,18 +284,18 @@ github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
 github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
-github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
-github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
+github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -321,10 +326,10 @@ github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j
 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
-github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
-github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
+github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -348,7 +353,7 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
 github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
 github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
+github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
 github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
 github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
 github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
@@ -366,8 +371,8 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo=
 github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4=
-github.com/nats-io/nats-server/v2 v2.10.5 h1:hhWt6m9ja/mNnm6ixc85jCthDaiUFPaeJI79K/MD980=
-github.com/nats-io/nats-server/v2 v2.10.5/go.mod h1:xUMTU4kS//SDkJCSvFwN9SyJ9nUuLhSkzB/Qz0dvjjg=
+github.com/nats-io/nats-server/v2 v2.10.7 h1:f5VDy+GMu7JyuFA0Fef+6TfulfCs5nBTgq7MMkFJx5Y=
+github.com/nats-io/nats-server/v2 v2.10.7/go.mod h1:V2JHOvPiPdtfDXTuEUsthUnCvSDeFrK4Xn9hRo6du7c=
 github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E=
 github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8=
 github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY=
@@ -477,14 +482,24 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
-go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
+go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo=
+go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
+go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
+go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
+go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
+go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -495,8 +510,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -555,15 +570,15 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -616,8 +631,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -634,8 +649,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY=
-golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -687,15 +702,15 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
 google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.151.0 h1:FhfXLO/NFdJIzQtCqjpysWwqKk8AzGWBUhMIx67cVDU=
-google.golang.org/api v0.151.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg=
+google.golang.org/api v0.154.0 h1:X7QkVKZBskztmpPKWQXgjJRPA2dJYrL6r+sYPRLj050=
+google.golang.org/api v0.154.0/go.mod h1:qhSMkM85hgqiokIYsrRyKxrjfBeIhgl4Z2JmeRkYylc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
-google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -714,12 +729,12 @@ google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA=
-google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI=
-google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k=
-google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 h1:AB/lmRny7e2pLhFEYIbl5qkDAUt2h0ZRO4wGPhZf+ik=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405/go.mod h1:67X1fPuzjcrkymZzZV1vvkFeTn2Rvc6lYF9MYFGCcwE=
+google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg=
+google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f/go.mod h1:nWSwAFPb+qfNJXsoeO3Io7zf4tMSfN8EA8RlDA04GhY=
+google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY=
+google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -729,8 +744,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
-google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
+google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU=
+google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
diff --git a/sdks/go/container/Dockerfile b/sdks/go/container/Dockerfile
index 30a9830cd195..7f1884cfb719 100644
--- a/sdks/go/container/Dockerfile
+++ b/sdks/go/container/Dockerfile
@@ -16,7 +16,7 @@
 # limitations under the License.
 ###############################################################################
 
-FROM debian:bullseye
+FROM debian:bookworm
 LABEL Author "Apache Beam <dev@beam.apache.org>"
 
 ARG TARGETOS
diff --git a/sdks/go/examples/timer_wordcap/wordcap.go b/sdks/go/examples/timer_wordcap/wordcap.go
index db64c10eb4d7..01f87edc59bd 100644
--- a/sdks/go/examples/timer_wordcap/wordcap.go
+++ b/sdks/go/examples/timer_wordcap/wordcap.go
@@ -118,7 +118,6 @@ func (s *Stateful) OnTimer(ctx context.Context, ts beam.EventTime, sp state.Prov
 			// Clean up the state that has been evicted.
 			s.ElementBag.Clear(sp)
 			s.MinTime.Clear(sp)
-			s.OutputState.ClearTag(tp, tag) // Clean up the fired timer tag. (Temporary workaround for a runner bug.)
 		}
 	}
 }
diff --git a/sdks/go/pkg/beam/core/core.go b/sdks/go/pkg/beam/core/core.go
index 07326d96528d..862b6eee94ba 100644
--- a/sdks/go/pkg/beam/core/core.go
+++ b/sdks/go/pkg/beam/core/core.go
@@ -27,7 +27,7 @@ const (
 	// SdkName is the human readable name of the SDK for UserAgents.
 	SdkName = "Apache Beam SDK for Go"
 	// SdkVersion is the current version of the SDK.
-	SdkVersion = "2.53.0.dev"
+	SdkVersion = "2.54.0.dev"
 
 	// DefaultDockerImage represents the associated image for this release.
 	DefaultDockerImage = "apache/beam_go_sdk:" + SdkVersion
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasampler.go b/sdks/go/pkg/beam/core/runtime/exec/datasampler.go
new file mode 100644
index 000000000000..15f32870b49c
--- /dev/null
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasampler.go
@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+	"sync"
+	"time"
+)
+
+// DataSample contains property for sampled element
+type DataSample struct {
+	PCollectionID string
+	Timestamp     time.Time
+	Element       []byte
+}
+
+// DataSampler manages sampled elements based on PCollectionID
+type DataSampler struct {
+	sampleChannel chan *DataSample
+	samplesMap    sync.Map // Key: PCollectionID string, Value: *OutputSamples pointer
+	ctx           context.Context
+}
+
+// NewDataSampler inits a new Data Sampler object and returns pointer to it.
+func NewDataSampler(ctx context.Context) *DataSampler {
+	return &DataSampler{
+		sampleChannel: make(chan *DataSample, 1000),
+		ctx:           ctx,
+	}
+}
+
+// Process processes sampled element.
+func (d *DataSampler) Process() {
+	for {
+		select {
+		case <-d.ctx.Done():
+			return
+		case sample := <-d.sampleChannel:
+			d.addSample(sample)
+		}
+	}
+}
+
+// GetSamples returns samples for given pCollectionID.
+// If no pCollectionID is provided, return all available samples
+func (d *DataSampler) GetSamples(pids []string) map[string][]*DataSample {
+	if len(pids) == 0 {
+		return d.getAllSamples()
+	}
+	return d.getSamplesForPCollections(pids)
+}
+
+// SendSample is called by PCollection Node to send sampled element to Data Sampler async
+func (d *DataSampler) SendSample(pCollectionID string, element []byte, timestamp time.Time) {
+	sample := DataSample{
+		PCollectionID: pCollectionID,
+		Element:       element,
+		Timestamp:     timestamp,
+	}
+	d.sampleChannel <- &sample
+}
+
+func (d *DataSampler) getAllSamples() map[string][]*DataSample {
+	var res = make(map[string][]*DataSample)
+	d.samplesMap.Range(func(key any, value any) bool {
+		pid := key.(string)
+		samples := d.getSamples(pid)
+		if len(samples) > 0 {
+			res[pid] = samples
+		}
+		return true
+	})
+	return res
+}
+
+func (d *DataSampler) getSamplesForPCollections(pids []string) map[string][]*DataSample {
+	var res = make(map[string][]*DataSample)
+	for _, pid := range pids {
+		samples := d.getSamples(pid)
+		if len(samples) > 0 {
+			res[pid] = samples
+		}
+	}
+	return res
+}
+
+func (d *DataSampler) addSample(sample *DataSample) {
+	p, ok := d.samplesMap.Load(sample.PCollectionID)
+	if !ok {
+		p = &outputSamples{maxElements: 10, sampleIndex: 0}
+		d.samplesMap.Store(sample.PCollectionID, p)
+	}
+	outputSamples := p.(*outputSamples)
+	outputSamples.addSample(sample)
+}
+
+func (d *DataSampler) getSamples(pCollectionID string) []*DataSample {
+	p, ok := d.samplesMap.Load(pCollectionID)
+	if !ok {
+		return nil
+	}
+	outputSamples := p.(*outputSamples)
+	return outputSamples.getSamples()
+}
+
+type outputSamples struct {
+	elements    []*DataSample
+	mu          sync.Mutex
+	maxElements int
+	sampleIndex int
+}
+
+func (o *outputSamples) addSample(element *DataSample) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
+	if len(o.elements) < o.maxElements {
+		o.elements = append(o.elements, element)
+	} else {
+		o.elements[o.sampleIndex] = element
+		o.sampleIndex = (o.sampleIndex + 1) % o.maxElements
+	}
+}
+
+func (o *outputSamples) getSamples() []*DataSample {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	if len(o.elements) == 0 {
+		return nil
+	}
+	samples := o.elements
+
+	// Reset index and samples
+	o.sampleIndex = 0
+	// Release memory since samples are only returned once based on best efforts
+	o.elements = nil
+
+	return samples
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasampler_test.go b/sdks/go/pkg/beam/core/runtime/exec/datasampler_test.go
new file mode 100644
index 000000000000..7dce38360afa
--- /dev/null
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasampler_test.go
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// TestDataSampler verifies that the DataSampler works correctly.
+func TestDataSampler(t *testing.T) {
+	timestamp := time.Now()
+	tests := []struct {
+		name    string
+		samples []DataSample
+		pids    []string
+		want    map[string][]*DataSample
+	}{
+		{
+			name: "GetAllSamples",
+			samples: []DataSample{
+				{PCollectionID: "pid1", Element: []byte("element1"), Timestamp: timestamp},
+				{PCollectionID: "pid2", Element: []byte("element2"), Timestamp: timestamp},
+			},
+			pids: []string{},
+			want: map[string][]*DataSample{
+				"pid1": {{PCollectionID: "pid1", Element: []byte("element1"), Timestamp: timestamp}},
+				"pid2": {{PCollectionID: "pid2", Element: []byte("element2"), Timestamp: timestamp}},
+			},
+		},
+		{
+			name: "GetSamplesForPCollections",
+			samples: []DataSample{
+				{PCollectionID: "pid1", Element: []byte("element1"), Timestamp: timestamp},
+				{PCollectionID: "pid2", Element: []byte("element2"), Timestamp: timestamp},
+			},
+			pids: []string{"pid1"},
+			want: map[string][]*DataSample{
+				"pid1": {{PCollectionID: "pid1", Element: []byte("element1"), Timestamp: timestamp}},
+			},
+		},
+		{
+			name: "GetSamplesForPCollectionsWithNoResult",
+			samples: []DataSample{
+				{PCollectionID: "pid1", Element: []byte("element1"), Timestamp: timestamp},
+				{PCollectionID: "pid2", Element: []byte("element2"), Timestamp: timestamp},
+			},
+			pids: []string{"pid3"},
+			want: map[string][]*DataSample{},
+		},
+		{
+			name: "GetSamplesForPCollectionsTooManySamples",
+			samples: []DataSample{
+				{PCollectionID: "pid1", Element: []byte("element1"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element2"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element3"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element4"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element5"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element6"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element7"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element8"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element9"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element10"), Timestamp: timestamp},
+				{PCollectionID: "pid1", Element: []byte("element11"), Timestamp: timestamp},
+			},
+			pids: []string{"pid1"},
+			want: map[string][]*DataSample{
+				"pid1": {
+					{PCollectionID: "pid1", Element: []byte("element2"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element3"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element4"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element5"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element6"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element7"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element8"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element9"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element10"), Timestamp: timestamp},
+					{PCollectionID: "pid1", Element: []byte("element11"), Timestamp: timestamp},
+				}},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			ctx, cancel := context.WithCancel(ctx)
+			dataSampler := NewDataSampler(ctx)
+			go dataSampler.Process()
+			for _, sample := range test.samples {
+				dataSampler.SendSample(sample.PCollectionID, sample.Element, sample.Timestamp)
+			}
+			var samplesCount = -1
+			var samples map[string][]*DataSample
+			for i := 0; i < 5; i++ {
+				samples = dataSampler.GetSamples(test.pids)
+				if len(samples) == len(test.want) {
+					samplesCount = len(samples)
+					break
+				}
+				time.Sleep(time.Second)
+			}
+			cancel()
+			if samplesCount != len(test.want) {
+				t.Errorf("got an unexpected number of sampled elements: %v, want: %v", samplesCount, len(test.want))
+			}
+			if !verifySampledElements(samples, test.want) {
+				t.Errorf("got an unexpected sampled elements: %v, want: %v", samples, test.want)
+			}
+		})
+	}
+}
+
+func verifySampledElements(samples, want map[string][]*DataSample) bool {
+	if len(samples) != len(want) {
+		return false
+	}
+	for pid, samples := range samples {
+		expected, ok := want[pid]
+		if !ok {
+			return false
+		}
+		sort.SliceStable(samples, func(i, j int) bool {
+			return string(samples[i].Element) < string(samples[j].Element)
+		})
+		sort.SliceStable(expected, func(i, j int) bool {
+			return string(expected[i].Element) < string(expected[j].Element)
+		})
+		if !reflect.DeepEqual(samples, expected) {
+			return false
+		}
+	}
+	return true
+}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasink.go b/sdks/go/pkg/beam/core/runtime/exec/datasink.go
index 6b39a2bb44f1..a58c93083f57 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasink.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasink.go
@@ -60,7 +60,7 @@ func (n *DataSink) StartBundle(ctx context.Context, id string, data DataContext)
 	// TODO[BEAM-6374): Properly handle the multiplex and flatten cases.
 	// Right now we just stop datasink collection.
 	if n.PCol != nil {
-		atomic.StoreInt64(&n.PCol.elementCount, 0)
+		atomic.StoreInt64(&n.PCol.bundleElementCount, 0)
 		n.PCol.resetSize()
 	}
 	return nil
@@ -86,7 +86,7 @@ func (n *DataSink) ProcessElement(ctx context.Context, value *FullValue, values
 	// TODO[BEAM-6374): Properly handle the multiplex and flatten cases.
 	// Right now we just stop datasink collection.
 	if n.PCol != nil {
-		atomic.AddInt64(&n.PCol.elementCount, 1)
+		atomic.AddInt64(&n.PCol.bundleElementCount, 1)
 		n.PCol.addSize(int64(byteCount))
 	}
 	return nil
diff --git a/sdks/go/pkg/beam/core/runtime/exec/datasource.go b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
index 401cdbef7a37..674de44cf35b 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/datasource.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/datasource.go
@@ -241,6 +241,7 @@ func (n *DataSource) Process(ctx context.Context) ([]*Checkpoint, error) {
 				return err
 			}
 			// Collect the actual size of the element, and reset the bytecounter reader.
+			// TODO(zechenj18) 2023-12-07: currently we never sample anything from the DataSource, we need to validate CoGBKs and similar types with the sampling implementation
 			n.PCol.addSize(int64(bcr.reset()))
 
 			// Check if there's a continuation and return residuals
diff --git a/sdks/go/pkg/beam/core/runtime/exec/pcollection.go b/sdks/go/pkg/beam/core/runtime/exec/pcollection.go
index 3b2e3ab3bf2c..6df2bd846532 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/pcollection.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/pcollection.go
@@ -16,12 +16,14 @@
 package exec
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"math"
 	"math/rand"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
 )
@@ -32,19 +34,23 @@ import (
 // In particular, must not be placed after a Multiplex, and must be placed
 // after a Flatten.
 type PCollection struct {
-	UID    UnitID
-	PColID string
-	Out    Node // Out is the consumer of this PCollection.
-	Coder  *coder.Coder
-	Seed   int64
+	UID         UnitID
+	PColID      string
+	Out         Node // Out is the consumer of this PCollection.
+	Coder       *coder.Coder
+	WindowCoder *coder.WindowCoder
+	Seed        int64
 
 	r             *rand.Rand
 	nextSampleIdx int64 // The index of the next value to sample.
 	elementCoder  ElementEncoder
+	windowCoder   WindowEncoder
 
-	elementCount                         int64 // must use atomic operations.
+	bundleElementCount                   int64 // must use atomic operations.
+	pCollectionElementCount              int64 // track the total number of elements this instance has processed. Local use only, no concurrent read/write.
 	sizeMu                               sync.Mutex
 	sizeCount, sizeSum, sizeMin, sizeMax int64
+	dataSampler                          *DataSampler
 }
 
 // ID returns the debug id for this unit.
@@ -57,12 +63,13 @@ func (p *PCollection) Up(ctx context.Context) error {
 	// dedicated rand source
 	p.r = rand.New(rand.NewSource(p.Seed))
 	p.elementCoder = MakeElementEncoder(p.Coder)
+	p.windowCoder = MakeWindowEncoder(p.WindowCoder)
 	return nil
 }
 
 // StartBundle resets collected metrics for this PCollection, and propagates bundle start.
 func (p *PCollection) StartBundle(ctx context.Context, id string, data DataContext) error {
-	atomic.StoreInt64(&p.elementCount, 0)
+	atomic.StoreInt64(&p.bundleElementCount, 0)
 	p.nextSampleIdx = 1
 	p.resetSize()
 	return MultiStartBundle(ctx, id, data, p.Out)
@@ -79,8 +86,8 @@ func (w *byteCounter) Write(p []byte) (n int, err error) {
 
 // ProcessElement increments the element count and sometimes takes size samples of the elements.
 func (p *PCollection) ProcessElement(ctx context.Context, elm *FullValue, values ...ReStream) error {
-	cur := atomic.AddInt64(&p.elementCount, 1)
-	if cur == p.nextSampleIdx {
+	cur := atomic.AddInt64(&p.bundleElementCount, 1)
+	if cur+p.pCollectionElementCount == p.nextSampleIdx {
 		// Always encode the first 3 elements. Otherwise...
 		// We pick the next sampling index based on how large this pcollection already is.
 		// We don't want to necessarily wait until the pcollection has doubled, so we reduce the range.
@@ -91,11 +98,21 @@ func (p *PCollection) ProcessElement(ctx context.Context, elm *FullValue, values
 		if p.nextSampleIdx < 4 {
 			p.nextSampleIdx++
 		} else {
-			p.nextSampleIdx = cur + p.r.Int63n(cur/10+2) + 1
+			p.nextSampleIdx = cur + p.r.Int63n((cur+p.pCollectionElementCount)/10+2) + 1
+		}
+
+		if p.dataSampler == nil {
+			var w byteCounter
+			p.elementCoder.Encode(elm, &w)
+			p.addSize(int64(w.count))
+		} else {
+			var buf bytes.Buffer
+			EncodeWindowedValueHeader(p.windowCoder, elm.Windows, elm.Timestamp, elm.Pane, &buf)
+			winSize := buf.Len()
+			p.elementCoder.Encode(elm, &buf)
+			p.addSize(int64(buf.Len() - winSize))
+			p.dataSampler.SendSample(p.PColID, buf.Bytes(), time.Now())
 		}
-		var w byteCounter
-		p.elementCoder.Encode(elm, &w)
-		p.addSize(int64(w.count))
 	}
 	return p.Out.ProcessElement(ctx, elm, values...)
 }
@@ -124,6 +141,7 @@ func (p *PCollection) resetSize() {
 
 // FinishBundle propagates bundle termination.
 func (p *PCollection) FinishBundle(ctx context.Context) error {
+	p.pCollectionElementCount += atomic.LoadInt64(&p.bundleElementCount)
 	return MultiFinishBundle(ctx, p.Out)
 }
 
@@ -149,7 +167,7 @@ func (p *PCollection) snapshot() PCollectionSnapshot {
 	defer p.sizeMu.Unlock()
 	return PCollectionSnapshot{
 		ID:           p.PColID,
-		ElementCount: atomic.LoadInt64(&p.elementCount),
+		ElementCount: atomic.LoadInt64(&p.bundleElementCount),
 		SizeCount:    p.sizeCount,
 		SizeSum:      p.sizeSum,
 		SizeMin:      p.sizeMin,
diff --git a/sdks/go/pkg/beam/core/runtime/exec/pcollection_test.go b/sdks/go/pkg/beam/core/runtime/exec/pcollection_test.go
index 1cb6adee97d6..1b702588c051 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/pcollection_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/pcollection_test.go
@@ -30,7 +30,7 @@ import (
 // randomness for the samples.
 func TestPCollection(t *testing.T) {
 	a := &CaptureNode{UID: 1}
-	pcol := &PCollection{UID: 2, Out: a, Coder: coder.NewVarInt()}
+	pcol := &PCollection{UID: 2, Out: a, Coder: coder.NewVarInt(), WindowCoder: coder.NewGlobalWindow()}
 	// The "large" 2nd value is to ensure the values are encoded properly,
 	// and that Min & Max are behaving.
 	inputs := []any{int64(1), int64(2000000000), int64(3)}
@@ -99,7 +99,7 @@ func BenchmarkPCollection(b *testing.B) {
 			Elm:       int64(1),
 		}})
 	}
-	pcol := &PCollection{UID: 2, Out: out, Coder: coder.NewVarInt()}
+	pcol := &PCollection{UID: 2, Out: out, Coder: coder.NewVarInt(), WindowCoder: coder.NewGlobalWindow()}
 	n := &FixedRoot{UID: 3, Elements: process, Out: pcol}
 	p, err := NewPlan("a", []Unit{n, pcol, out})
 	if err != nil {
diff --git a/sdks/go/pkg/beam/core/runtime/exec/to_string.go b/sdks/go/pkg/beam/core/runtime/exec/to_string.go
index 2196fd951806..df7050483b16 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/to_string.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/to_string.go
@@ -45,6 +45,7 @@ func (m *ToString) ProcessElement(ctx context.Context, elm *FullValue, values ..
 		Elm:       elm.Elm,
 		Elm2:      fmt.Sprintf("%v", elm.Elm2),
 		Timestamp: elm.Timestamp,
+		Pane:      elm.Pane,
 	}
 
 	return m.Out.ProcessElement(ctx, &ret, values...)
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate.go b/sdks/go/pkg/beam/core/runtime/exec/translate.go
index 6b3e3e457229..72af9e80c405 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate.go
@@ -51,8 +51,8 @@ const (
 )
 
 // UnmarshalPlan converts a model bundle descriptor into an execution Plan.
-func UnmarshalPlan(desc *fnpb.ProcessBundleDescriptor) (*Plan, error) {
-	b, err := newBuilder(desc)
+func UnmarshalPlan(desc *fnpb.ProcessBundleDescriptor, dataSampler *DataSampler) (*Plan, error) {
+	b, err := newBuilder(desc, dataSampler)
 	if err != nil {
 		return nil, err
 	}
@@ -169,8 +169,9 @@ type builder struct {
 	nodes     map[string]*PCollection // PCollectionID -> Node (cache)
 	links     map[linkID]Node         // linkID -> Node (cache)
 
-	units []Unit // result
-	idgen *GenID
+	units       []Unit // result
+	idgen       *GenID
+	dataSampler *DataSampler
 }
 
 // linkID represents an incoming data link to an Node.
@@ -179,7 +180,7 @@ type linkID struct {
 	input int    // input index. If > 0, it's a side input.
 }
 
-func newBuilder(desc *fnpb.ProcessBundleDescriptor) (*builder, error) {
+func newBuilder(desc *fnpb.ProcessBundleDescriptor, dataSampler *DataSampler) (*builder, error) {
 	// Preprocess graph structure to allow insertion of Multiplex,
 	// Flatten and Discard.
 
@@ -216,7 +217,8 @@ func newBuilder(desc *fnpb.ProcessBundleDescriptor) (*builder, error) {
 		nodes:     make(map[string]*PCollection),
 		links:     make(map[linkID]Node),
 
-		idgen: &GenID{},
+		idgen:       &GenID{},
+		dataSampler: dataSampler,
 	}
 	return b, nil
 }
@@ -411,11 +413,11 @@ func (b *builder) makePCollection(id string) (*PCollection, error) {
 }
 
 func (b *builder) newPCollectionNode(id string, out Node) (*PCollection, error) {
-	ec, _, err := b.makeCoderForPCollection(id)
+	ec, wc, err := b.makeCoderForPCollection(id)
 	if err != nil {
 		return nil, err
 	}
-	u := &PCollection{UID: b.idgen.New(), Out: out, PColID: id, Coder: ec, Seed: rand.Int63()}
+	u := &PCollection{UID: b.idgen.New(), Out: out, PColID: id, Coder: ec, WindowCoder: wc, Seed: rand.Int63(), dataSampler: b.dataSampler}
 	b.nodes[id] = u
 	b.units = append(b.units, u)
 	return u, nil
@@ -525,7 +527,7 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
 					u = &TruncateSizedRestriction{UID: b.idgen.New(), Fn: dofn, Out: out[0]}
 				default:
 					n := &ParDo{UID: b.idgen.New(), Fn: dofn, Inbound: in, Out: out}
-					n.PID = transform.GetUniqueName()
+					n.PID = id.to
 
 					input := unmarshalKeyedValues(transform.GetInputs())
 
@@ -658,7 +660,7 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
 				}
 				cn.UsesKey = typex.IsKV(in[0].Type)
 
-				cn.PID = transform.GetUniqueName()
+				cn.PID = id.to
 
 				switch urn {
 				case urnPerKeyCombinePre:
@@ -796,7 +798,7 @@ func (b *builder) makeLink(from string, id linkID) (Node, error) {
 		if err != nil {
 			return nil, err
 		}
-		u = &MapWindows{UID: b.idgen.New(), Fn: mapper, Out: out[0]}
+		u = &MapWindows{UID: b.idgen.New(), Fn: mapper, Out: out[0], FnUrn: fn.GetUrn()}
 
 	case graphx.URNFlatten:
 		u = &Flatten{UID: b.idgen.New(), N: len(transform.Inputs), Out: out[0]}
diff --git a/sdks/go/pkg/beam/core/runtime/exec/translate_test.go b/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
index c6a70fe07a02..a9917ec456fe 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/translate_test.go
@@ -460,7 +460,7 @@ func TestUnmarshalPlan(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			plan, err := UnmarshalPlan(test.inputDesc)
+			plan, err := UnmarshalPlan(test.inputDesc, nil)
 			if err != nil && test.outputError == nil {
 				t.Errorf("there is an error where should not be. UnmarshalPlan(%v) = (%v, %v), want (%v, %v)", test.inputDesc, plan, err, test.outputPlan, test.outputError)
 			} else if err != nil && !reflect.DeepEqual(err, test.outputError) {
@@ -503,7 +503,7 @@ func TestNewBuilder(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			b, err := newBuilder(test.inputDesc)
+			b, err := newBuilder(test.inputDesc, nil)
 			if err != nil && test.outputError == nil {
 				t.Errorf("There is an error where should not be. newBuilder(%v) = (%v, %v), want (%v, %v)", test.inputDesc, b, err, test.outputBuilder, test.outputError)
 			} else if err != nil && err != test.outputError {
diff --git a/sdks/go/pkg/beam/core/runtime/exec/window.go b/sdks/go/pkg/beam/core/runtime/exec/window.go
index 081fed48ac62..fabd6af933d8 100644
--- a/sdks/go/pkg/beam/core/runtime/exec/window.go
+++ b/sdks/go/pkg/beam/core/runtime/exec/window.go
@@ -105,6 +105,8 @@ type MapWindows struct {
 	UID UnitID
 	Fn  WindowMapper
 	Out Node
+
+	FnUrn string // Keep the urn for debugging purposes.
 }
 
 // ID returns the UnitID for this unit.
@@ -122,11 +124,17 @@ func (m *MapWindows) StartBundle(ctx context.Context, id string, data DataContex
 }
 
 func (m *MapWindows) ProcessElement(ctx context.Context, elm *FullValue, values ...ReStream) error {
-	w, ok := elm.Elm2.(window.IntervalWindow)
-	if !ok {
-		return errors.Errorf("not an IntervalWindow, got %T", elm.Elm2)
+	// MapWindows ends up with the wrappedDecode path, which can pass the value window through the
+	// Window field. Use that as the default for resilience to a change to match the coder correctly.
+	win := elm.Windows[0]
+	if elm.Elm2 != nil {
+		w, ok := elm.Elm2.(typex.Window)
+		if !ok {
+			return errors.Errorf("not a Window Value, got %T", elm.Elm2)
+		}
+		win = w
 	}
-	newW, err := m.Fn.MapWindow(w)
+	newW, err := m.Fn.MapWindow(win)
 	if err != nil {
 		return err
 	}
@@ -151,7 +159,7 @@ func (m *MapWindows) Down(_ context.Context) error {
 }
 
 func (m *MapWindows) String() string {
-	return fmt.Sprintf("MapWindows[%v]. Out:%v", m.Fn, m.Out.ID())
+	return fmt.Sprintf("MapWindows[%v]. Out:%v", m.FnUrn, m.Out.ID())
 }
 
 // WindowMapper defines an interface maps windows from a main input window space
diff --git a/sdks/go/pkg/beam/core/runtime/graphx/translate.go b/sdks/go/pkg/beam/core/runtime/graphx/translate.go
index 9ef28eb7809b..3b7cfc5639cd 100644
--- a/sdks/go/pkg/beam/core/runtime/graphx/translate.go
+++ b/sdks/go/pkg/beam/core/runtime/graphx/translate.go
@@ -73,6 +73,7 @@ const (
 	URNMultiCore             = "beam:protocol:multi_core_bundle_processing:v1"
 	URNWorkerStatus          = "beam:protocol:worker_status:v1"
 	URNMonitoringInfoShortID = "beam:protocol:monitoring_info_short_ids:v1"
+	URNDataSampling          = "beam:protocol:data_sampling:v1"
 
 	URNRequiresSplittableDoFn     = "beam:requirement:pardo:splittable_dofn:v1"
 	URNRequiresBundleFinalization = "beam:requirement:pardo:finalization:v1"
@@ -109,6 +110,7 @@ func goCapabilities() []string {
 		URNMonitoringInfoShortID,
 		URNBaseVersionGo,
 		URNToString,
+		URNDataSampling,
 	}
 	return append(capabilities, knownStandardCoders()...)
 }
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
index ed57e3eca59b..0f2de99dd2ad 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr.go
@@ -386,14 +386,31 @@ func (c *DataChannel) read(ctx context.Context) {
 			return
 		}
 
+		// Consolidating required timer writer creation to a optional single lock section.
+		type seenTimers struct {
+			InstID                 instructionID
+			PTransformID, FamilyID string
+		}
+		neededTimers := map[seenTimers]struct{}{}
+
 		// Each message may contain segments for multiple streams, so we
 		// must treat each segment in isolation. We maintain a local cache
 		// to reduce lock contention.
 		iterateElements(c, cache, &seenLast, msg.GetTimers(),
 			func(elm *fnpb.Elements_Timers) exec.Elements {
+				neededTimers[seenTimers{InstID: instructionID(elm.GetInstructionId()), PTransformID: elm.GetTransformId(), FamilyID: elm.GetTimerFamilyId()}] = struct{}{}
 				return exec.Elements{Timers: elm.GetTimers(), PtransformID: elm.GetTransformId(), TimerFamilyID: elm.GetTimerFamilyId()}
 			})
 
+		// Creating a writer is necessary to ensure a "is_last" signal is returned for timers that aren't set.
+		if len(neededTimers) > 0 {
+			c.mu.Lock()
+			for key := range neededTimers {
+				c.makeTimerWriterLocked(ctx, clientID{ptransformID: key.PTransformID, instID: key.InstID}, key.FamilyID)
+			}
+			c.mu.Unlock()
+		}
+
 		iterateElements(c, cache, &seenLast, msg.GetData(),
 			func(elm *fnpb.Elements_Data) exec.Elements {
 				return exec.Elements{Data: elm.GetData(), PtransformID: elm.GetTransformId()}
@@ -629,7 +646,13 @@ func (w *dataWriter) Write(p []byte) (n int, err error) {
 func (c *DataChannel) makeTimerWriter(ctx context.Context, id clientID, family string) *timerWriter {
 	c.mu.Lock()
 	defer c.mu.Unlock()
+	return c.makeTimerWriterLocked(ctx, id, family)
+}
 
+// makeTimerWriterLocked does the work of makeTimerWriter, but doesn't call the lock methods.
+//
+// c.mu must be locked when this is called.
+func (c *DataChannel) makeTimerWriterLocked(ctx context.Context, id clientID, family string) *timerWriter {
 	var m map[timerKey]*timerWriter
 	var ok bool
 	if m, ok = c.timerWriters[id.instID]; !ok {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
index c7f8ac5858c1..92c4d0a8f8cd 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/datamgr_test.go
@@ -389,6 +389,18 @@ func TestElementChan(t *testing.T) {
 				return elms
 			},
 			wantSum: 0, wantCount: 0,
+		}, {
+			name: "SomeTimersAndADataThenReaderThenCleanup",
+			sequenceFn: func(ctx context.Context, t *testing.T, client *fakeChanClient, c *DataChannel) <-chan exec.Elements {
+				client.Send(&fnpb.Elements{
+					Timers: []*fnpb.Elements_Timers{timerElm(1, false), timerElm(2, true)},
+					Data:   []*fnpb.Elements_Data{dataElm(3, true)},
+				})
+				elms := openChan(ctx, t, c, timerID)
+				c.removeInstruction(instID)
+				return elms
+			},
+			wantSum: 6, wantCount: 3,
 		},
 	}
 	for _, test := range tests {
diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness.go b/sdks/go/pkg/beam/core/runtime/harness/harness.go
index c5db9a85f367..2888b39853e3 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/harness.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/harness.go
@@ -30,6 +30,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/metrics"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/harness/statecache"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/hooks"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/internal/errors"
@@ -40,6 +41,7 @@ import (
 	"golang.org/x/sync/singleflight"
 	"google.golang.org/grpc"
 	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 // URNMonitoringInfoShortID is a URN indicating support for short monitoring info IDs.
@@ -157,6 +159,11 @@ func MainWithOptions(ctx context.Context, loggingEndpoint, controlEndpoint strin
 		runnerCapabilities:   rcMap,
 	}
 
+	if enabled, ok := rcMap[graphx.URNDataSampling]; ok && enabled {
+		ctrl.dataSampler = exec.NewDataSampler(ctx)
+		go ctrl.dataSampler.Process()
+	}
+
 	// if the runner supports worker status api then expose SDK harness status
 	if opts.StatusEndpoint != "" {
 		statusHandler, err := newWorkerStatusHandler(ctx, opts.StatusEndpoint, ctrl.cache, func(statusInfo *strings.Builder) { ctrl.metStoreToString(statusInfo) })
@@ -304,6 +311,7 @@ type control struct {
 	// TODO(BEAM-11097): Cache is currently unused.
 	cache              *statecache.SideInputCache
 	runnerCapabilities map[string]bool
+	dataSampler        *exec.DataSampler
 }
 
 func (c *control) metStoreToString(statusInfo *strings.Builder) {
@@ -345,7 +353,7 @@ func (c *control) getOrCreatePlan(bdID bundleDescriptorID) (*exec.Plan, error) {
 		}
 		desc = newDesc.(*fnpb.ProcessBundleDescriptor)
 	}
-	newPlan, err := exec.UnmarshalPlan(desc)
+	newPlan, err := exec.UnmarshalPlan(desc, c.dataSampler)
 	if err != nil {
 		return nil, errors.WithContextf(err, "invalid bundle desc: %v\n%v\n", bdID, desc.String())
 	}
@@ -654,7 +662,30 @@ func (c *control) handleInstruction(ctx context.Context, req *fnpb.InstructionRe
 				},
 			},
 		}
+	case req.GetSampleData() != nil:
+		msg := req.GetSampleData()
+		var samples = make(map[string]*fnpb.SampleDataResponse_ElementList)
+		if c.dataSampler != nil {
+			var elementsMap = c.dataSampler.GetSamples(msg.GetPcollectionIds())
+			for pid, elements := range elementsMap {
+				var elementList fnpb.SampleDataResponse_ElementList
+				for i := range elements {
+					var sampledElement = &fnpb.SampledElement{
+						Element:         elements[i].Element,
+						SampleTimestamp: timestamppb.New(elements[i].Timestamp),
+					}
+					elementList.Elements = append(elementList.Elements, sampledElement)
+				}
+				samples[pid] = &elementList
+			}
+		}
 
+		return &fnpb.InstructionResponse{
+			InstructionId: string(instID),
+			Response: &fnpb.InstructionResponse_SampleData{
+				SampleData: &fnpb.SampleDataResponse{ElementSamples: samples},
+			},
+		}
 	default:
 		return fail(ctx, instID, "Unexpected request: %v", req)
 	}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/harness_test.go b/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
index 84c5770c71a1..91dd3c591d5b 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/harness_test.go
@@ -94,7 +94,7 @@ func invalidDescriptor(t *testing.T) *fnpb.ProcessBundleDescriptor {
 
 func TestControl_getOrCreatePlan(t *testing.T) {
 	testBDID := bundleDescriptorID("test")
-	testPlan, err := exec.UnmarshalPlan(validDescriptor(t))
+	testPlan, err := exec.UnmarshalPlan(validDescriptor(t), nil)
 	if err != nil {
 		t.Fatal("bad testPlan")
 	}
diff --git a/sdks/go/pkg/beam/core/runtime/harness/init/init.go b/sdks/go/pkg/beam/core/runtime/harness/init/init.go
index 55f3d0beabdc..27d44f7029e5 100644
--- a/sdks/go/pkg/beam/core/runtime/harness/init/init.go
+++ b/sdks/go/pkg/beam/core/runtime/harness/init/init.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"encoding/json"
 	"flag"
+	"slices"
 	"strings"
 	"time"
 
@@ -31,6 +32,7 @@ import (
 	"runtime/debug"
 
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/harness"
 
 	// Import gcs filesystem so that it can be used to upload heap dumps.
@@ -78,6 +80,13 @@ func hook() {
 		return
 	}
 
+	// Extract environment variables. These are optional runner supported capabilities.
+	// Expected env variables:
+	// RUNNER_CAPABILITIES : list of runner supported capability urn.
+	// STATUS_ENDPOINT : Endpoint to connect to status server used for worker status reporting.
+	statusEndpoint := os.Getenv("STATUS_ENDPOINT")
+	runnerCapabilities := strings.Split(os.Getenv("RUNNER_CAPABILITIES"), " ")
+
 	// Initialization logging
 	//
 	// We use direct output to stderr here, because it is expected that logging
@@ -91,6 +100,14 @@ func hook() {
 			os.Exit(1)
 		}
 		runtime.GlobalOptions.Import(opt.Options)
+		var experiments []string
+		if e, ok := opt.Options.Options["experiments"]; ok {
+			experiments = strings.Split(e, ",")
+		}
+		// TODO(zechenj18) 2023-12-07: Remove once the data sampling URN is properly sent in via the capabilities
+		if slices.Contains(experiments, "enable_data_sampling") {
+			runnerCapabilities = append(runnerCapabilities, graphx.URNDataSampling)
+		}
 	}
 
 	defer func() {
@@ -120,12 +137,6 @@ func hook() {
 		fmt.Println("Error Setting Rlimit ", err)
 	}
 
-	// Extract environment variables. These are optional runner supported capabilities.
-	// Expected env variables:
-	// RUNNER_CAPABILITIES : list of runner supported capability urn.
-	// STATUS_ENDPOINT : Endpoint to connect to status server used for worker status reporting.
-	statusEndpoint := os.Getenv("STATUS_ENDPOINT")
-	runnerCapabilities := strings.Split(os.Getenv("RUNNER_CAPABILITIES"), " ")
 	options := harness.Options{
 		StatusEndpoint:     statusEndpoint,
 		RunnerCapabilities: runnerCapabilities,
diff --git a/sdks/go/pkg/beam/core/typex/fulltype.go b/sdks/go/pkg/beam/core/typex/fulltype.go
index 41ef0ab09d22..ff5520c28617 100644
--- a/sdks/go/pkg/beam/core/typex/fulltype.go
+++ b/sdks/go/pkg/beam/core/typex/fulltype.go
@@ -124,7 +124,7 @@ func New(t reflect.Type, components ...FullType) FullType {
 			if len(components) != 2 {
 				panic(fmt.Sprintf("Invalid number of components for KV: %v, %v", t, components))
 			}
-			if isAnyNonKVComposite(components) {
+			if isAnyNonKVAndNonWindowedComposite(components) {
 				panic(fmt.Sprintf("Invalid to nest composite composites inside KV: %v, %v", t, components))
 			}
 			return &tree{class, t, components}
@@ -169,6 +169,15 @@ func isAnyNonKVComposite(list []FullType) bool {
 	return false
 }
 
+func isAnyNonKVAndNonWindowedComposite(list []FullType) bool {
+	for _, t := range list {
+		if t.Class() == Composite && t.Type() != KVType && t.Type() != WindowedValueType {
+			return true
+		}
+	}
+	return false
+}
+
 // Convenience functions.
 
 // IsW returns true iff the type is a WindowedValue.
diff --git a/sdks/go/pkg/beam/io/natsio/common.go b/sdks/go/pkg/beam/io/natsio/common.go
index 53f595516987..72640894c76e 100644
--- a/sdks/go/pkg/beam/io/natsio/common.go
+++ b/sdks/go/pkg/beam/io/natsio/common.go
@@ -31,6 +31,10 @@ type natsFn struct {
 }
 
 func (fn *natsFn) Setup() error {
+	if fn.nc != nil && fn.js != nil {
+		return nil
+	}
+
 	var opts []nats.Option
 	if fn.CredsFile != "" {
 		opts = append(opts, nats.UserCredentials(fn.CredsFile))
diff --git a/sdks/go/pkg/beam/io/natsio/end_estimator.go b/sdks/go/pkg/beam/io/natsio/end_estimator.go
new file mode 100644
index 000000000000..6b2f18e10ce3
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/end_estimator.go
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+type endEstimator struct {
+	js      jetstream.JetStream
+	stream  string
+	subject string
+}
+
+func newEndEstimator(js jetstream.JetStream, stream string, subject string) *endEstimator {
+	return &endEstimator{
+		js:      js,
+		stream:  stream,
+		subject: subject,
+	}
+}
+
+func (e *endEstimator) Estimate() int64 {
+	ctx := context.Background()
+	end, err := e.getEndSeqNo(ctx)
+	if err != nil {
+		panic(err)
+	}
+	return end
+}
+
+func (e *endEstimator) getEndSeqNo(ctx context.Context) (int64, error) {
+	str, err := e.js.Stream(ctx, e.stream)
+	if err != nil {
+		return -1, fmt.Errorf("error getting stream: %v", err)
+	}
+
+	msg, err := str.GetLastMsgForSubject(ctx, e.subject)
+	if err != nil {
+		if isMessageNotFound(err) {
+			return 1, nil
+		}
+
+		return -1, fmt.Errorf("error getting last message: %v", err)
+	}
+
+	return int64(msg.Sequence) + 1, nil
+}
+
+func isMessageNotFound(err error) bool {
+	var jsErr jetstream.JetStreamError
+	if errors.As(err, &jsErr) {
+		apiErr := jsErr.APIError()
+		if apiErr.ErrorCode == jetstream.JSErrCodeMessageNotFound {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/sdks/go/pkg/beam/io/natsio/end_estimator_test.go b/sdks/go/pkg/beam/io/natsio/end_estimator_test.go
new file mode 100644
index 000000000000..855547a0e297
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/end_estimator_test.go
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/nats-io/nats.go"
+)
+
+func Test_endEstimator_Estimate(t *testing.T) {
+	tests := []struct {
+		name    string
+		msgs    []*nats.Msg
+		subject string
+		want    int64
+	}{
+		{
+			name: "Estimate end for published messages",
+			msgs: []*nats.Msg{
+				{
+					Subject: "subject.1",
+					Data:    []byte("msg1"),
+				},
+				{
+					Subject: "subject.1",
+					Data:    []byte("msg2"),
+				},
+				{
+					Subject: "subject.2",
+					Data:    []byte("msg3"),
+				},
+			},
+			subject: "subject.1",
+			want:    3,
+		},
+		{
+			name:    "Estimate end for no published messages",
+			subject: "subject.1",
+			want:    1,
+		},
+	}
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			srv := newServer(t)
+			url := srv.ClientURL()
+			conn := newConn(t, url)
+			js := newJetStream(t, conn)
+
+			stream := fmt.Sprintf("STREAM-%d", i)
+			subjectFilter := "subject.*"
+
+			createStream(ctx, t, js, stream, []string{subjectFilter})
+			publishMessages(ctx, t, js, tt.msgs)
+
+			estimator := newEndEstimator(js, stream, tt.subject)
+			if got := estimator.Estimate(); got != tt.want {
+				t.Fatalf("Estimate() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/sdks/go/pkg/beam/io/natsio/example_test.go b/sdks/go/pkg/beam/io/natsio/example_test.go
index 0516b8efa921..984261a3686b 100644
--- a/sdks/go/pkg/beam/io/natsio/example_test.go
+++ b/sdks/go/pkg/beam/io/natsio/example_test.go
@@ -22,9 +22,27 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/io/natsio"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/x/debug"
 	"github.com/nats-io/nats.go"
 )
 
+func ExampleRead() {
+	beam.Init()
+
+	p, s := beam.NewPipelineWithRoot()
+
+	uri := "nats://localhost:4222"
+	stream := "EVENTS"
+	subject := "events.*"
+
+	col := natsio.Read(s, uri, stream, subject)
+	debug.Print(s, col)
+
+	if err := beamx.Run(context.Background(), p); err != nil {
+		log.Fatalf("Failed to execute job: %v", err)
+	}
+}
+
 func ExampleWrite() {
 	beam.Init()
 
diff --git a/sdks/go/pkg/beam/io/natsio/helper_test.go b/sdks/go/pkg/beam/io/natsio/helper_test.go
index cd47ed331de0..ac7eedac1d44 100644
--- a/sdks/go/pkg/beam/io/natsio/helper_test.go
+++ b/sdks/go/pkg/beam/io/natsio/helper_test.go
@@ -18,6 +18,7 @@ package natsio
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats-server/v2/test"
@@ -62,8 +63,8 @@ func newJetStream(t *testing.T, conn *nats.Conn) jetstream.JetStream {
 }
 
 func createStream(
-	t *testing.T,
 	ctx context.Context,
+	t *testing.T,
 	js jetstream.JetStream,
 	stream string,
 	subjects []string,
@@ -89,8 +90,8 @@ func createStream(
 }
 
 func createConsumer(
-	t *testing.T,
 	ctx context.Context,
+	t *testing.T,
 	js jetstream.JetStream,
 	stream string,
 	subjects []string,
@@ -128,3 +129,46 @@ func fetchMessages(t *testing.T, cons jetstream.Consumer, size int) []jetstream.
 
 	return result
 }
+
+func publishMessages(ctx context.Context, t *testing.T, js jetstream.JetStream, msgs []*nats.Msg) {
+	t.Helper()
+
+	for _, msg := range msgs {
+		if _, err := js.PublishMsg(ctx, msg); err != nil {
+			t.Fatalf("Failed to publish message: %v", err)
+		}
+	}
+}
+
+func messagesWithPublishingTime(
+	t *testing.T,
+	pubMsgs []jetstream.Msg,
+	pubIndices []int,
+	want []any,
+) []any {
+	t.Helper()
+
+	wantWTime := make([]any, len(want))
+
+	for i := range want {
+		pubIdx := pubIndices[i]
+		pubMsg := pubMsgs[pubIdx]
+
+		wantMsg := want[i].(ConsumerMessage)
+		wantMsg.PublishingTime = messageTimestamp(t, pubMsg)
+		wantWTime[i] = wantMsg
+	}
+
+	return wantWTime
+}
+
+func messageTimestamp(t *testing.T, msg jetstream.Msg) time.Time {
+	t.Helper()
+
+	metadata, err := msg.Metadata()
+	if err != nil {
+		t.Fatalf("Failed to retrieve metadata: %v", err)
+	}
+
+	return metadata.Timestamp
+}
diff --git a/sdks/go/pkg/beam/io/natsio/read.go b/sdks/go/pkg/beam/io/natsio/read.go
new file mode 100644
index 000000000000..df5a53accbe1
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/read.go
@@ -0,0 +1,289 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"reflect"
+	"time"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/sdf"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/io/rtrackers/offsetrange"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/register"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+func init() {
+	register.DoFn5x2[
+		context.Context, *watermarkEstimator, *sdf.LockRTracker, []byte,
+		func(beam.EventTime, ConsumerMessage), sdf.ProcessContinuation, error,
+	](
+		&readFn{},
+	)
+	register.Emitter2[beam.EventTime, ConsumerMessage]()
+	beam.RegisterType(reflect.TypeOf((*ConsumerMessage)(nil)).Elem())
+}
+
+const (
+	defaultFetchSize  = 100
+	defaultStartSeqNo = 1
+	defaultEndSeqNo   = math.MaxInt64
+	fetchTimeout      = 3 * time.Second
+	assumedLag        = 1 * time.Second
+	resumeDelay       = 5 * time.Second
+)
+
+type ConsumerMessage struct {
+	Subject        string
+	PublishingTime time.Time
+	ID             string
+	Headers        map[string][]string
+	Data           []byte
+}
+
+// Read reads messages from NATS JetStream and returns a PCollection<ConsumerMessage>.
+// Read takes a variable number of ReadOptionFn to configure the read operation:
+//   - UserCredentials: path to the user credentials file. Defaults to empty.
+//   - ProcessingTimePolicy: whether to use the pipeline processing time of the messages as the event
+//     time. Defaults to true.
+//   - PublishingTimePolicy: whether to use the publishing time of the messages as the event time.
+//     Defaults to false.
+//   - FetchSize: the maximum number of messages to retrieve at a time. Defaults to 100.
+//   - StartSeqNo: the start sequence number of messages to read. Defaults to 1.
+//   - EndSeqNo: the end sequence number of messages to read (exclusive). Defaults to math.MaxInt64.
+func Read(
+	s beam.Scope,
+	uri string,
+	stream string,
+	subject string,
+	opts ...ReadOptionFn,
+) beam.PCollection {
+	s = s.Scope("natsio.Read")
+
+	option := &readOption{
+		TimePolicy: processingTimePolicy,
+		FetchSize:  defaultFetchSize,
+		StartSeqNo: defaultStartSeqNo,
+		EndSeqNo:   defaultEndSeqNo,
+	}
+
+	for _, opt := range opts {
+		if err := opt(option); err != nil {
+			panic(fmt.Sprintf("natsio.Read: invalid option: %v", err))
+		}
+	}
+
+	imp := beam.Impulse(s)
+	return beam.ParDo(s, newReadFn(uri, stream, subject, option), imp)
+}
+
+type readFn struct {
+	natsFn
+	Stream      string
+	Subject     string
+	TimePolicy  timePolicy
+	FetchSize   int
+	StartSeqNo  int64
+	EndSeqNo    int64
+	timestampFn timestampFn
+}
+
+func newReadFn(uri string, stream string, subject string, option *readOption) *readFn {
+	return &readFn{
+		natsFn: natsFn{
+			URI:       uri,
+			CredsFile: option.CredsFile,
+		},
+		Stream:     stream,
+		Subject:    subject,
+		TimePolicy: option.TimePolicy,
+		FetchSize:  option.FetchSize,
+		StartSeqNo: option.StartSeqNo,
+		EndSeqNo:   option.EndSeqNo,
+	}
+}
+
+func (fn *readFn) Setup() error {
+	if err := fn.natsFn.Setup(); err != nil {
+		return err
+	}
+
+	fn.timestampFn = fn.TimePolicy.TimestampFn()
+	return nil
+}
+
+func (fn *readFn) CreateInitialRestriction(_ []byte) offsetrange.Restriction {
+	return offsetrange.Restriction{
+		Start: fn.StartSeqNo,
+		End:   fn.EndSeqNo,
+	}
+}
+
+func (fn *readFn) SplitRestriction(
+	_ []byte,
+	rest offsetrange.Restriction,
+) []offsetrange.Restriction {
+	return []offsetrange.Restriction{rest}
+}
+
+func (fn *readFn) RestrictionSize(_ []byte, rest offsetrange.Restriction) (float64, error) {
+	if err := fn.natsFn.Setup(); err != nil {
+		return -1, err
+	}
+
+	rt, err := fn.createRTracker(rest)
+	if err != nil {
+		return -1, err
+	}
+
+	_, remaining := rt.GetProgress()
+	return remaining, nil
+}
+
+func (fn *readFn) CreateTracker(rest offsetrange.Restriction) (*sdf.LockRTracker, error) {
+	rt, err := fn.createRTracker(rest)
+	if err != nil {
+		return nil, err
+	}
+
+	return sdf.NewLockRTracker(rt), nil
+}
+
+func (fn *readFn) TruncateRestriction(rt *sdf.LockRTracker, _ []byte) offsetrange.Restriction {
+	start := rt.GetRestriction().(offsetrange.Restriction).Start
+	return offsetrange.Restriction{
+		Start: start,
+		End:   start,
+	}
+}
+
+func (fn *readFn) InitialWatermarkEstimatorState(
+	et beam.EventTime,
+	_ offsetrange.Restriction,
+	_ []byte,
+) int64 {
+	return et.Milliseconds()
+}
+
+func (fn *readFn) CreateWatermarkEstimator(ms int64) *watermarkEstimator {
+	return &watermarkEstimator{state: ms}
+}
+
+func (fn *readFn) WatermarkEstimatorState(we *watermarkEstimator) int64 {
+	return we.state
+}
+
+func (fn *readFn) ProcessElement(
+	ctx context.Context,
+	we *watermarkEstimator,
+	rt *sdf.LockRTracker,
+	_ []byte,
+	emit func(beam.EventTime, ConsumerMessage),
+) (sdf.ProcessContinuation, error) {
+	startSeqNo := rt.GetRestriction().(offsetrange.Restriction).Start
+	cons, err := fn.createConsumer(ctx, startSeqNo)
+	if err != nil {
+		return sdf.StopProcessing(), err
+	}
+
+	for {
+		msgs, err := cons.Fetch(fn.FetchSize, jetstream.FetchMaxWait(fetchTimeout))
+		if err != nil {
+			return nil, fmt.Errorf("error fetching messages: %v", err)
+		}
+
+		count := 0
+		for msg := range msgs.Messages() {
+			metadata, err := msg.Metadata()
+			if err != nil {
+				return sdf.StopProcessing(), fmt.Errorf("error retrieving metadata: %v", err)
+			}
+
+			seqNo := int64(metadata.Sequence.Stream)
+			if !rt.TryClaim(seqNo) {
+				return sdf.StopProcessing(), nil
+			}
+
+			et := fn.timestampFn(metadata.Timestamp)
+			consMsg := createConsumerMessage(msg, metadata.Timestamp)
+			emit(et, consMsg)
+
+			count++
+		}
+
+		if err := msgs.Error(); err != nil {
+			return sdf.StopProcessing(), fmt.Errorf("error in message batch: %v", err)
+		}
+
+		if count == 0 {
+			fn.updateWatermarkManually(we)
+			return sdf.ResumeProcessingIn(resumeDelay), nil
+		}
+	}
+}
+
+func (fn *readFn) createRTracker(rest offsetrange.Restriction) (sdf.RTracker, error) {
+	if rest.End < math.MaxInt64 {
+		return offsetrange.NewTracker(rest), nil
+	}
+
+	estimator := newEndEstimator(fn.js, fn.Stream, fn.Subject)
+	rt, err := offsetrange.NewGrowableTracker(rest, estimator)
+	if err != nil {
+		return nil, fmt.Errorf("error creating growable tracker: %v", err)
+	}
+
+	return rt, nil
+}
+
+func (fn *readFn) createConsumer(
+	ctx context.Context,
+	startSeqNo int64,
+) (jetstream.Consumer, error) {
+	cfg := jetstream.OrderedConsumerConfig{
+		FilterSubjects:   []string{fn.Subject},
+		DeliverPolicy:    jetstream.DeliverByStartSequencePolicy,
+		OptStartSeq:      uint64(startSeqNo),
+		MaxResetAttempts: 5,
+	}
+
+	cons, err := fn.js.OrderedConsumer(ctx, fn.Stream, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("error creating consumer: %v", err)
+	}
+
+	return cons, nil
+}
+
+func createConsumerMessage(msg jetstream.Msg, publishingTime time.Time) ConsumerMessage {
+	return ConsumerMessage{
+		Subject:        msg.Subject(),
+		PublishingTime: publishingTime,
+		ID:             msg.Headers().Get(nats.MsgIdHdr),
+		Headers:        msg.Headers(),
+		Data:           msg.Data(),
+	}
+}
+
+func (fn *readFn) updateWatermarkManually(we *watermarkEstimator) {
+	t := time.Now().Add(-1 * assumedLag)
+	et := fn.timestampFn(t)
+	we.ObserveTimestamp(et.ToTime())
+}
diff --git a/sdks/go/pkg/beam/io/natsio/read_option.go b/sdks/go/pkg/beam/io/natsio/read_option.go
new file mode 100644
index 000000000000..f9d715be1022
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/read_option.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import "errors"
+
+var (
+	errInvalidFetchSize  = errors.New("fetch size must be greater than 0")
+	errInvalidStartSeqNo = errors.New("start sequence number must be greater than 0")
+	errInvalidEndSeqNo   = errors.New("end sequence number must be greater than 0")
+)
+
+type readOption struct {
+	CredsFile  string
+	TimePolicy timePolicy
+	FetchSize  int
+	StartSeqNo int64
+	EndSeqNo   int64
+}
+
+// ReadOptionFn is a function that can be passed to Read to configure options for reading
+// from NATS.
+type ReadOptionFn func(option *readOption) error
+
+// ReadUserCredentials sets the user credentials when connecting to NATS.
+func ReadUserCredentials(credsFile string) ReadOptionFn {
+	return func(o *readOption) error {
+		o.CredsFile = credsFile
+		return nil
+	}
+}
+
+// ReadProcessingTimePolicy specifies that the pipeline processing time of the messages should be
+// used to compute the watermark estimate.
+func ReadProcessingTimePolicy() ReadOptionFn {
+	return func(o *readOption) error {
+		o.TimePolicy = processingTimePolicy
+		return nil
+	}
+}
+
+// ReadPublishingTimePolicy specifies that the publishing time of the messages should be used to
+// compute the watermark estimate.
+func ReadPublishingTimePolicy() ReadOptionFn {
+	return func(o *readOption) error {
+		o.TimePolicy = publishingTimePolicy
+		return nil
+	}
+}
+
+// ReadFetchSize sets the maximum number of messages to retrieve at a time.
+func ReadFetchSize(size int) ReadOptionFn {
+	return func(o *readOption) error {
+		if size <= 0 {
+			return errInvalidFetchSize
+		}
+
+		o.FetchSize = size
+		return nil
+	}
+}
+
+// ReadStartSeqNo sets the start sequence number of messages to read.
+func ReadStartSeqNo(seqNo int64) ReadOptionFn {
+	return func(o *readOption) error {
+		if seqNo <= 0 {
+			return errInvalidStartSeqNo
+		}
+
+		o.StartSeqNo = seqNo
+		return nil
+	}
+}
+
+// ReadEndSeqNo sets the end sequence number of messages to read (exclusive).
+func ReadEndSeqNo(seqNo int64) ReadOptionFn {
+	return func(o *readOption) error {
+		if seqNo <= 0 {
+			return errInvalidEndSeqNo
+		}
+
+		o.EndSeqNo = seqNo
+		return nil
+	}
+}
diff --git a/sdks/go/pkg/beam/io/natsio/read_test.go b/sdks/go/pkg/beam/io/natsio/read_test.go
new file mode 100644
index 000000000000..faf0b0540c81
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/read_test.go
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest"
+	"github.com/nats-io/nats.go"
+)
+
+func TestRead(t *testing.T) {
+	tests := []struct {
+		name       string
+		input      []*nats.Msg
+		subject    string
+		opts       []ReadOptionFn
+		pubIndices []int
+		want       []any
+	}{
+		{
+			name: "Read messages from bounded stream with single subject",
+			input: []*nats.Msg{
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"123"}},
+					Data:    []byte("msg1"),
+				},
+				{
+					Subject: "subject.2",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"124"}},
+					Data:    []byte("msg2"),
+				},
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"125"}},
+					Data:    []byte("msg3"),
+				},
+			},
+			subject: "subject.1",
+			opts: []ReadOptionFn{
+				ReadEndSeqNo(4),
+			},
+			pubIndices: []int{0, 2},
+			want: []any{
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "123",
+					Headers: map[string][]string{nats.MsgIdHdr: {"123"}},
+					Data:    []byte("msg1"),
+				},
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "125",
+					Headers: map[string][]string{nats.MsgIdHdr: {"125"}},
+					Data:    []byte("msg3"),
+				},
+			},
+		},
+		{
+			name: "Read messages from bounded stream with wildcard subject",
+			input: []*nats.Msg{
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"123"}},
+					Data:    []byte("msg1"),
+				},
+				{
+					Subject: "subject.2",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"124"}},
+					Data:    []byte("msg2"),
+				},
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"125"}},
+					Data:    []byte("msg3"),
+				},
+			},
+			subject: "subject.*",
+			opts: []ReadOptionFn{
+				ReadEndSeqNo(4),
+			},
+			pubIndices: []int{0, 1, 2},
+			want: []any{
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "123",
+					Headers: map[string][]string{nats.MsgIdHdr: {"123"}},
+					Data:    []byte("msg1"),
+				},
+				ConsumerMessage{
+					Subject: "subject.2",
+					ID:      "124",
+					Headers: map[string][]string{nats.MsgIdHdr: {"124"}},
+					Data:    []byte("msg2"),
+				},
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "125",
+					Headers: map[string][]string{nats.MsgIdHdr: {"125"}},
+					Data:    []byte("msg3"),
+				},
+			},
+		},
+		{
+			name: "Read messages from bounded stream with custom fetch size",
+			input: []*nats.Msg{
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"123"}},
+					Data:    []byte("msg1"),
+				},
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"124"}},
+					Data:    []byte("msg2"),
+				},
+			},
+			subject: "subject.1",
+			opts: []ReadOptionFn{
+				ReadFetchSize(1),
+				ReadEndSeqNo(3),
+			},
+			pubIndices: []int{0, 1},
+			want: []any{
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "123",
+					Headers: map[string][]string{nats.MsgIdHdr: {"123"}},
+					Data:    []byte("msg1"),
+				},
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "124",
+					Headers: map[string][]string{nats.MsgIdHdr: {"124"}},
+					Data:    []byte("msg2"),
+				},
+			},
+		},
+		{
+			name: "Read messages from bounded stream with custom start seq no",
+			input: []*nats.Msg{
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"123"}},
+					Data:    []byte("msg1"),
+				},
+				{
+					Subject: "subject.1",
+					Header:  nats.Header{nats.MsgIdHdr: []string{"124"}},
+					Data:    []byte("msg2"),
+				},
+			},
+			subject: "subject.1",
+			opts: []ReadOptionFn{
+				ReadStartSeqNo(2),
+				ReadEndSeqNo(3),
+			},
+			pubIndices: []int{1},
+			want: []any{
+				ConsumerMessage{
+					Subject: "subject.1",
+					ID:      "124",
+					Headers: map[string][]string{nats.MsgIdHdr: {"124"}},
+					Data:    []byte("msg2"),
+				},
+			},
+		},
+	}
+	for i, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			srv := newServer(t)
+			url := srv.ClientURL()
+			conn := newConn(t, url)
+			js := newJetStream(t, conn)
+
+			stream := fmt.Sprintf("STREAM-%d", i)
+			subjectFilter := "subject.*"
+
+			createStream(ctx, t, js, stream, []string{subjectFilter})
+			publishMessages(ctx, t, js, tt.input)
+
+			cons := createConsumer(ctx, t, js, stream, []string{subjectFilter})
+			pubMsgs := fetchMessages(t, cons, len(tt.input))
+			wantWTime := messagesWithPublishingTime(t, pubMsgs, tt.pubIndices, tt.want)
+
+			p, s := beam.NewPipelineWithRoot()
+			got := Read(s, url, stream, tt.subject, tt.opts...)
+
+			passert.Equals(s, got, wantWTime...)
+			ptest.RunAndValidate(t, p)
+		})
+	}
+}
diff --git a/sdks/go/pkg/beam/io/natsio/time_policy.go b/sdks/go/pkg/beam/io/natsio/time_policy.go
new file mode 100644
index 000000000000..1c2dbf5165f7
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/time_policy.go
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"time"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
+)
+
+type timePolicy int
+
+const (
+	processingTimePolicy timePolicy = iota
+	publishingTimePolicy
+)
+
+type timestampFn func(time.Time) mtime.Time
+
+func processingTime(_ time.Time) mtime.Time {
+	return mtime.Now()
+}
+
+func publishingTime(t time.Time) mtime.Time {
+	return mtime.FromTime(t)
+}
+
+func (p timePolicy) TimestampFn() timestampFn {
+	switch p {
+	case processingTimePolicy:
+		return processingTime
+	case publishingTimePolicy:
+		return publishingTime
+	default:
+		panic("unsupported time policy")
+	}
+}
diff --git a/sdks/go/pkg/beam/io/natsio/time_policy_test.go b/sdks/go/pkg/beam/io/natsio/time_policy_test.go
new file mode 100644
index 000000000000..2452334c8cdd
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/time_policy_test.go
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"testing"
+	"time"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
+)
+
+func Test_timePolicy_TimestampFn(t *testing.T) {
+	t.Run("processingTime", func(t *testing.T) {
+		pubTime := time.Date(2020, 1, 2, 3, 4, 5, 6e6, time.UTC)
+
+		t1 := mtime.Now()
+		got := processingTimePolicy.TimestampFn()(pubTime)
+		t2 := mtime.Now()
+
+		if got < t1 || got > t2 {
+			t.Errorf("timestamp = %v, want between %v and %v", got, t1, t2)
+		}
+	})
+
+	t.Run("publishingTime", func(t *testing.T) {
+		pubTime := time.Date(2020, 1, 2, 3, 4, 5, 6e6, time.UTC)
+
+		if got, want := publishingTimePolicy.TimestampFn()(pubTime), mtime.FromTime(pubTime); got != want {
+			t.Errorf("timestamp = %v, want %v", got, want)
+		}
+	})
+}
diff --git a/sdks/go/pkg/beam/io/natsio/watermark_estimator.go b/sdks/go/pkg/beam/io/natsio/watermark_estimator.go
new file mode 100644
index 000000000000..b23eb37ac855
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/watermark_estimator.go
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import "time"
+
+type watermarkEstimator struct {
+	state int64
+}
+
+func (e *watermarkEstimator) CurrentWatermark() time.Time {
+	return time.UnixMilli(e.state)
+}
+
+func (e *watermarkEstimator) ObserveTimestamp(t time.Time) {
+	ms := t.UnixMilli()
+	if ms > e.state {
+		e.state = ms
+	}
+}
diff --git a/sdks/go/pkg/beam/io/natsio/watermark_estimator_test.go b/sdks/go/pkg/beam/io/natsio/watermark_estimator_test.go
new file mode 100644
index 000000000000..91a9a840a6e0
--- /dev/null
+++ b/sdks/go/pkg/beam/io/natsio/watermark_estimator_test.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package natsio
+
+import (
+	"testing"
+	"time"
+)
+
+func Test_watermarkEstimator_CurrentWatermark(t *testing.T) {
+	ms := int64(1577934245000)
+	we := &watermarkEstimator{
+		state: ms,
+	}
+	if got, want := we.CurrentWatermark(), time.UnixMilli(ms); got != want {
+		t.Errorf("CurrentWatermark() = %v, want %v", got, want)
+	}
+}
+
+func Test_watermarkEstimator_ObserveTimestamp(t *testing.T) {
+	t1 := time.Date(2020, 1, 2, 3, 4, 5, 6e6, time.UTC)
+	t2 := time.Date(2020, 1, 2, 3, 4, 5, 7e6, time.UTC)
+
+	tests := []struct {
+		name  string
+		state int64
+		t     time.Time
+		want  int64
+	}{
+		{
+			name:  "Update watermark when the time is greater than the current state",
+			state: t1.UnixMilli(),
+			t:     t2,
+			want:  t2.UnixMilli(),
+		},
+		{
+			name:  "Keep existing watermark when the time is not greater than the current state",
+			state: t2.UnixMilli(),
+			t:     t1,
+			want:  t2.UnixMilli(),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			we := &watermarkEstimator{
+				state: tt.state,
+			}
+			we.ObserveTimestamp(tt.t)
+			if got, want := we.state, tt.want; got != want {
+				t.Errorf("state = %v, want %v", got, want)
+			}
+		})
+	}
+}
diff --git a/sdks/go/pkg/beam/io/natsio/write_test.go b/sdks/go/pkg/beam/io/natsio/write_test.go
index 5e9387ece5f6..678874ce11e9 100644
--- a/sdks/go/pkg/beam/io/natsio/write_test.go
+++ b/sdks/go/pkg/beam/io/natsio/write_test.go
@@ -187,8 +187,8 @@ func TestWrite(t *testing.T) {
 			js := newJetStream(t, conn)
 
 			subjects := []string{subject}
-			createStream(t, ctx, js, stream, subjects)
-			cons := createConsumer(t, ctx, js, stream, subjects)
+			createStream(ctx, t, js, stream, subjects)
+			cons := createConsumer(ctx, t, js, stream, subjects)
 
 			p, s := beam.NewPipelineWithRoot()
 
diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow.go b/sdks/go/pkg/beam/runners/dataflow/dataflow.go
index 7b43ba78f054..ca701979497a 100644
--- a/sdks/go/pkg/beam/runners/dataflow/dataflow.go
+++ b/sdks/go/pkg/beam/runners/dataflow/dataflow.go
@@ -35,6 +35,7 @@ import (
 
 	"cloud.google.com/go/storage"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/graphx"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/pipelinex"
@@ -442,7 +443,17 @@ func getContainerImage(ctx context.Context) string {
 		if *image != "" {
 			return *image
 		}
-		return jobopts.GetEnvironmentConfig(ctx)
+		envConfig := jobopts.GetEnvironmentConfig(ctx)
+		if envConfig == core.DefaultDockerImage {
+			// It's possible the user set the image exactly manually, but unlikely.
+			// Prefer using the gcr.io image by default.
+			// Note: This doesn't change the dev experience, which requires a user
+			// to have a dev image.
+			// However, RC versions should automatically be picked up, since
+			// they are never tagged the RC number, just the main version.
+			return "gcr.io/cloud-dataflow/v1beta3/beam_go_sdk:" + core.SdkVersion
+		}
+		return envConfig
 	}
 	panic(fmt.Sprintf("Unsupported environment %v", urn))
 }
diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go b/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go
index 663695f00c8e..2e0f28f263d6 100644
--- a/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go
+++ b/sdks/go/pkg/beam/runners/dataflow/dataflow_test.go
@@ -22,6 +22,7 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/options/gcpopts"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/options/jobopts"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/dataflow/dataflowlib"
@@ -313,6 +314,15 @@ func TestGetJobOptions_DockerNoImage(t *testing.T) {
 	}
 }
 
+func TestGetJobOptions_DockerGCROverride(t *testing.T) {
+	resetGlobals()
+	*jobopts.EnvironmentType = "docker"
+
+	if got, want := getContainerImage(context.Background()), "gcr.io/cloud-dataflow/v1beta3/beam_go_sdk:"+core.SdkVersion; got != want {
+		t.Fatalf("getContainerImage() = %q, want %q", got, want)
+	}
+}
+
 func TestGetJobOptions_TransformMapping(t *testing.T) {
 	resetGlobals()
 	*stagingLocation = "gs://testStagingLocation"
diff --git a/sdks/go/pkg/beam/runners/dataflow/dataflowlib/job.go b/sdks/go/pkg/beam/runners/dataflow/dataflowlib/job.go
index eb26071d10ec..ed706ec1a482 100644
--- a/sdks/go/pkg/beam/runners/dataflow/dataflowlib/job.go
+++ b/sdks/go/pkg/beam/runners/dataflow/dataflowlib/job.go
@@ -154,6 +154,7 @@ func Translate(ctx context.Context, p *pipepb.Pipeline, opts *JobOptions, worker
 		return nil, err
 	}
 
+	opts.Options.Options["experiments"] = strings.Join(opts.Experiments, ",")
 	job := &df.Job{
 		ProjectId: opts.Project,
 		Name:      opts.Name,
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/data.go b/sdks/go/pkg/beam/runners/prism/internal/engine/data.go
index 6fc192ac83be..6679f484aa2d 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/data.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/data.go
@@ -15,10 +15,30 @@
 
 package engine
 
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/coder"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/window"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/runtime/exec"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
+	"golang.org/x/exp/slog"
+)
+
+// StateData is a "union" between Bag state and MultiMap state to increase common code.
+type StateData struct {
+	Bag      [][]byte
+	Multimap map[string][][]byte
+}
+
 // TentativeData is where data for in progress bundles is put
 // until the bundle executes successfully.
 type TentativeData struct {
 	Raw map[string][][]byte
+
+	// state is a map from transformID + UserStateID, to window, to userKey, to datavalues.
+	state map[LinkID]map[typex.Window]map[string]StateData
 }
 
 // WriteData adds data to a given global collectionID.
@@ -28,3 +48,159 @@ func (d *TentativeData) WriteData(colID string, data []byte) {
 	}
 	d.Raw[colID] = append(d.Raw[colID], data)
 }
+
+func (d *TentativeData) toWindow(wKey []byte) typex.Window {
+	if len(wKey) == 0 {
+		return window.GlobalWindow{}
+	}
+	// TODO: Custom Window handling.
+	w, err := exec.MakeWindowDecoder(coder.NewIntervalWindow()).DecodeSingle(bytes.NewBuffer(wKey))
+	if err != nil {
+		panic(fmt.Sprintf("error decoding append bag user state window key %v: %v", wKey, err))
+	}
+	return w
+}
+
+// GetBagState retrieves available state from the tentative bundle data.
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) GetBagState(stateID LinkID, wKey, uKey []byte) [][]byte {
+	winMap := d.state[stateID]
+	w := d.toWindow(wKey)
+	data := winMap[w][string(uKey)]
+	slog.Debug("State() Bag.Get", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("Window", w), slog.Any("Data", data))
+	return data.Bag
+}
+
+func (d *TentativeData) appendState(stateID LinkID, wKey []byte) map[string]StateData {
+	if d.state == nil {
+		d.state = map[LinkID]map[typex.Window]map[string]StateData{}
+	}
+	winMap, ok := d.state[stateID]
+	if !ok {
+		winMap = map[typex.Window]map[string]StateData{}
+		d.state[stateID] = winMap
+	}
+	w := d.toWindow(wKey)
+	kmap, ok := winMap[w]
+	if !ok {
+		kmap = map[string]StateData{}
+		winMap[w] = kmap
+	}
+	return kmap
+}
+
+// AppendBagState appends the incoming data to the existing tentative data bundle.
+//
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) AppendBagState(stateID LinkID, wKey, uKey, data []byte) {
+	kmap := d.appendState(stateID, wKey)
+	kmap[string(uKey)] = StateData{Bag: append(kmap[string(uKey)].Bag, data)}
+	slog.Debug("State() Bag.Append", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("Window", wKey), slog.Any("NewData", data))
+}
+
+func (d *TentativeData) clearState(stateID LinkID, wKey []byte) map[string]StateData {
+	if d.state == nil {
+		return nil
+	}
+	winMap, ok := d.state[stateID]
+	if !ok {
+		return nil
+	}
+	w := d.toWindow(wKey)
+	return winMap[w]
+}
+
+// ClearBagState clears any tentative data for the state. Since state data is only initialized if any exists,
+// Clear takes the approach to not create state that doesn't already exist. Existing state is zeroed
+// to allow that to be committed post bundle commpletion.
+//
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) ClearBagState(stateID LinkID, wKey, uKey []byte) {
+	kmap := d.clearState(stateID, wKey)
+	if kmap == nil {
+		return
+	}
+	// Zero the current entry to clear.
+	// Delete makes it difficult to delete the persisted stage state for the key.
+	kmap[string(uKey)] = StateData{}
+	slog.Debug("State() Bag.Clear", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("WindowKey", wKey))
+}
+
+// GetMultimapState retrieves available state from the tentative bundle data.
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) GetMultimapState(stateID LinkID, wKey, uKey, mapKey []byte) [][]byte {
+	winMap := d.state[stateID]
+	w := d.toWindow(wKey)
+	data := winMap[w][string(uKey)].Multimap[string(mapKey)]
+	slog.Debug("State() Multimap.Get", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("Window", w), slog.Any("Data", data))
+	return data
+}
+
+// AppendMultimapState appends the incoming data to the existing tentative data bundle.
+//
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) AppendMultimapState(stateID LinkID, wKey, uKey, mapKey, data []byte) {
+	kmap := d.appendState(stateID, wKey)
+	stateData, ok := kmap[string(uKey)]
+	if !ok || stateData.Multimap == nil { // Incase of All Key Clear tombstones, we may have a nil map.
+		stateData = StateData{Multimap: map[string][][]byte{}}
+		kmap[string(uKey)] = stateData
+	}
+	stateData.Multimap[string(mapKey)] = append(stateData.Multimap[string(mapKey)], data)
+	// The Multimap field is aliased to the instance we stored in kmap,
+	// so we don't need to re-assign back to kmap after appending the data to mapKey.
+	slog.Debug("State() Multimap.Append", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("MapKey", mapKey), slog.Any("Window", wKey), slog.Any("NewData", data))
+}
+
+// ClearMultimapState clears any tentative data for the state. Since state data is only initialized if any exists,
+// Clear takes the approach to not create state that doesn't already exist. Existing state is zeroed
+// to allow that to be committed post bundle commpletion.
+//
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) ClearMultimapState(stateID LinkID, wKey, uKey, mapKey []byte) {
+	kmap := d.clearState(stateID, wKey)
+	if kmap == nil {
+		return
+	}
+	// Nil the current entry to clear.
+	// Delete makes it difficult to delete the persisted stage state for the key.
+	userMap, ok := kmap[string(uKey)]
+	if !ok || userMap.Multimap == nil {
+		return
+	}
+	userMap.Multimap[string(mapKey)] = nil
+	// The Multimap field is aliased to the instance we stored in kmap,
+	// so we don't need to re-assign back to kmap after clearing the data from mapKey.
+	slog.Debug("State() Multimap.Clear", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("Window", wKey))
+}
+
+// GetMultimapKeysState retrieves all available user map keys.
+//
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) GetMultimapKeysState(stateID LinkID, wKey, uKey []byte) [][]byte {
+	winMap := d.state[stateID]
+	w := d.toWindow(wKey)
+	userMap := winMap[w][string(uKey)]
+	var keys [][]byte
+	for k := range userMap.Multimap {
+		keys = append(keys, []byte(k))
+	}
+	slog.Debug("State() MultimapKeys.Get", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("Window", w), slog.Any("Keys", keys))
+	return keys
+}
+
+// ClearMultimapKeysState clears tentative data for all user map keys. Since state data is only initialized if any exists,
+// Clear takes the approach to not create state that doesn't already exist. Existing state is zeroed
+// to allow that to be committed post bundle commpletion.
+//
+// The stateID has the Transform and Local fields populated, for the Transform and UserStateID respectively.
+func (d *TentativeData) ClearMultimapKeysState(stateID LinkID, wKey, uKey []byte) {
+	kmap := d.clearState(stateID, wKey)
+	if kmap == nil {
+		return
+	}
+	// Zero the current entry to clear.
+	// Delete makes it difficult to delete the persisted stage state for the key.
+	kmap[string(uKey)] = StateData{}
+	slog.Debug("State() MultimapKeys.Clear", slog.Any("StateID", stateID), slog.Any("UserKey", uKey), slog.Any("WindowKey", wKey))
+}
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
index 656525c67047..6cb552354186 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/elementmanager.go
@@ -23,6 +23,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -39,6 +40,7 @@ type element struct {
 	pane      typex.PaneInfo
 
 	elmBytes []byte
+	keyBytes []byte
 }
 
 type elements struct {
@@ -51,6 +53,7 @@ type PColInfo struct {
 	WDec     exec.WindowDecoder
 	WEnc     exec.WindowEncoder
 	EDec     func(io.Reader) []byte
+	KeyDec   func(io.Reader) []byte
 }
 
 // ToData recodes the elements with their approprate windowed value header.
@@ -182,6 +185,12 @@ func (em *ElementManager) StageAggregates(ID string) {
 	em.stages[ID].aggregate = true
 }
 
+// StageStateful marks the given stage as stateful, which means elements are
+// processed by key.
+func (em *ElementManager) StageStateful(ID string) {
+	em.stages[ID].stateful = true
+}
+
 // Impulse marks and initializes the given stage as an impulse which
 // is a root transform that starts processing.
 func (em *ElementManager) Impulse(stageID string) {
@@ -257,10 +266,13 @@ func (em *ElementManager) Bundles(ctx context.Context, nextBundID func() string)
 				ss := em.stages[stageID]
 				watermark, ready := ss.bundleReady(em)
 				if ready {
-					bundleID, ok := ss.startBundle(watermark, nextBundID)
+					bundleID, ok, reschedule := ss.startBundle(watermark, nextBundID)
 					if !ok {
 						continue
 					}
+					if reschedule {
+						em.watermarkRefreshes.insert(stageID)
+					}
 					rb := RunBundle{StageID: stageID, BundleID: bundleID, Watermark: watermark}
 
 					em.inprogressBundles.insert(rb.BundleID)
@@ -278,7 +290,11 @@ func (em *ElementManager) Bundles(ctx context.Context, nextBundID func() string)
 				v := em.livePending.Load()
 				slog.Debug("Bundles: nothing in progress and no refreshes", slog.Int64("pendingElementCount", v))
 				if v > 0 {
-					panic(fmt.Sprintf("nothing in progress and no refreshes with non zero pending elements: %v", v))
+					var stageState []string
+					for id, ss := range em.stages {
+						stageState = append(stageState, fmt.Sprintln(id, ss.pending, ss.pendingByKeys, ss.inprogressKeys, ss.inprogressKeysByBundle))
+					}
+					panic(fmt.Sprintf("nothing in progress and no refreshes with non zero pending elements: %v\n%v", v, strings.Join(stageState, "")))
 				}
 			} else if len(em.inprogressBundles) == 0 {
 				v := em.livePending.Load()
@@ -304,6 +320,56 @@ func (em *ElementManager) InputForBundle(rb RunBundle, info PColInfo) [][]byte {
 	return es.ToData(info)
 }
 
+// StateForBundle retreives relevant state for the given bundle, WRT the data in the bundle.
+//
+// TODO(lostluck): Consider unifiying with InputForBundle, to reduce lock contention.
+func (em *ElementManager) StateForBundle(rb RunBundle) TentativeData {
+	ss := em.stages[rb.StageID]
+	ss.mu.Lock()
+	defer ss.mu.Unlock()
+	var ret TentativeData
+	keys := ss.inprogressKeysByBundle[rb.BundleID]
+	// TODO(lostluck): Also track windows per bundle, to reduce copying.
+	if len(ss.state) > 0 {
+		ret.state = map[LinkID]map[typex.Window]map[string]StateData{}
+	}
+	for link, winMap := range ss.state {
+		for w, keyMap := range winMap {
+			for key := range keys {
+				data, ok := keyMap[key]
+				if !ok {
+					continue
+				}
+				linkMap, ok := ret.state[link]
+				if !ok {
+					linkMap = map[typex.Window]map[string]StateData{}
+					ret.state[link] = linkMap
+				}
+				wlinkMap, ok := linkMap[w]
+				if !ok {
+					wlinkMap = map[string]StateData{}
+					linkMap[w] = wlinkMap
+				}
+				var mm map[string][][]byte
+				if len(data.Multimap) > 0 {
+					mm = map[string][][]byte{}
+					for uk, v := range data.Multimap {
+						// Clone the "holding" slice, but refer to the existing data bytes.
+						mm[uk] = append([][]byte(nil), v...)
+					}
+				}
+				// Clone the "holding" slice, but refer to the existing data bytes.
+				wlinkMap[key] = StateData{
+					Bag:      append([][]byte(nil), data.Bag...),
+					Multimap: mm,
+				}
+			}
+		}
+	}
+
+	return ret
+}
+
 // reElementResiduals extracts the windowed value header from residual bytes, and explodes them
 // back out to their windows.
 func reElementResiduals(residuals [][]byte, inputInfo PColInfo, rb RunBundle) []element {
@@ -322,6 +388,15 @@ func reElementResiduals(residuals [][]byte, inputInfo PColInfo, rb RunBundle) []
 			slog.Error("reElementResiduals: sdk provided a windowed value header 0 windows", "bundle", rb)
 			panic("error decoding residual header: sdk provided a windowed value header 0 windows")
 		}
+		// POSSIBLY BAD PATTERN: The buffer is invalidated on the next call, which doesn't always happen.
+		// But the decoder won't be mutating the buffer bytes, just reading the data. So the elmBytes
+		// should remain pointing to the whole element, and we should have a copy of the key bytes.
+		// Ideally, we're simply refering to the key part of the existing buffer.
+		elmBytes := buf.Bytes()
+		var keyBytes []byte
+		if inputInfo.KeyDec != nil {
+			keyBytes = inputInfo.KeyDec(buf)
+		}
 
 		for _, w := range ws {
 			unprocessedElements = append(unprocessedElements,
@@ -329,7 +404,8 @@ func reElementResiduals(residuals [][]byte, inputInfo PColInfo, rb RunBundle) []
 					window:    w,
 					timestamp: et,
 					pane:      pn,
-					elmBytes:  buf.Bytes(),
+					elmBytes:  elmBytes,
+					keyBytes:  keyBytes,
 				})
 		}
 	}
@@ -373,6 +449,11 @@ func (em *ElementManager) PersistBundle(rb RunBundle, col2Coders map[string]PCol
 				}
 				// TODO: Optimize unnecessary copies. This is doubleteeing.
 				elmBytes := info.EDec(tee)
+				var keyBytes []byte
+				if info.KeyDec != nil {
+					kbuf := bytes.NewBuffer(elmBytes)
+					keyBytes = info.KeyDec(kbuf) // TODO: Optimize unnecessary copies. This is tripleteeing?
+				}
 				for _, w := range ws {
 					newPending = append(newPending,
 						element{
@@ -380,6 +461,7 @@ func (em *ElementManager) PersistBundle(rb RunBundle, col2Coders map[string]PCol
 							timestamp: et,
 							pane:      pn,
 							elmBytes:  elmBytes,
+							keyBytes:  keyBytes,
 						})
 				}
 			}
@@ -412,6 +494,10 @@ func (em *ElementManager) PersistBundle(rb RunBundle, col2Coders map[string]PCol
 	completed := stage.inprogress[rb.BundleID]
 	em.addPending(-len(completed.es))
 	delete(stage.inprogress, rb.BundleID)
+	for k := range stage.inprogressKeysByBundle[rb.BundleID] {
+		delete(stage.inprogressKeys, k)
+	}
+	delete(stage.inprogressKeysByBundle, rb.BundleID)
 	// If there are estimated output watermarks, set the estimated
 	// output watermark for the stage.
 	if len(estimatedOWM) > 0 {
@@ -421,6 +507,25 @@ func (em *ElementManager) PersistBundle(rb RunBundle, col2Coders map[string]PCol
 		}
 		stage.estimatedOutput = estimate
 	}
+
+	// Handle persisting.
+	for link, winMap := range d.state {
+		linkMap, ok := stage.state[link]
+		if !ok {
+			linkMap = map[typex.Window]map[string]StateData{}
+			stage.state[link] = linkMap
+		}
+		for w, keyMap := range winMap {
+			wlinkMap, ok := linkMap[w]
+			if !ok {
+				wlinkMap = map[string]StateData{}
+				linkMap[w] = wlinkMap
+			}
+			for key, data := range keyMap {
+				wlinkMap[key] = data
+			}
+		}
+	}
 	stage.mu.Unlock()
 
 	// TODO support state/timer watermark holds.
@@ -499,6 +604,11 @@ func (em *ElementManager) refreshWatermarks() set[string] {
 
 type set[K comparable] map[K]struct{}
 
+func (s set[K]) present(k K) bool {
+	_, ok := s[k]
+	return ok
+}
+
 func (s set[K]) remove(k K) {
 	delete(s, k)
 }
@@ -525,7 +635,8 @@ type stageState struct {
 	sides     []LinkID // PCollection IDs of side inputs that can block execution.
 
 	// Special handling bits
-	aggregate bool     // whether this state needs to block for aggregation.
+	stateful  bool     // whether this stage uses state or timers, and needs keyed processing.
+	aggregate bool     // whether this stage needs to block for aggregation.
 	strat     winStrat // Windowing Strategy for aggregation fireings.
 
 	mu                 sync.Mutex
@@ -537,6 +648,12 @@ type stageState struct {
 	pending    elementHeap                          // pending input elements for this stage that are to be processesd
 	inprogress map[string]elements                  // inprogress elements by active bundles, keyed by bundle
 	sideInputs map[LinkID]map[typex.Window][][]byte // side input data for this stage, from {tid, inputID} -> window
+
+	// Fields for stateful stages which need to be per key.
+	pendingByKeys          map[string]elementHeap                           // pending input elements by Key, if stateful.
+	inprogressKeys         set[string]                                      // all keys that are assigned to bundles.
+	inprogressKeysByBundle map[string]set[string]                           // bundle to key assignments.
+	state                  map[LinkID]map[typex.Window]map[string]StateData // state data for this stage, from {tid, stateID} -> window -> userKey
 }
 
 // makeStageState produces an initialized stageState.
@@ -546,6 +663,7 @@ func makeStageState(ID string, inputIDs, outputIDs []string, sides []LinkID) *st
 		outputIDs: outputIDs,
 		sides:     sides,
 		strat:     defaultStrat{},
+		state:     map[LinkID]map[typex.Window]map[string]StateData{},
 
 		input:           mtime.MinTimestamp,
 		output:          mtime.MinTimestamp,
@@ -566,8 +684,22 @@ func makeStageState(ID string, inputIDs, outputIDs []string, sides []LinkID) *st
 func (ss *stageState) AddPending(newPending []element) {
 	ss.mu.Lock()
 	defer ss.mu.Unlock()
-	ss.pending = append(ss.pending, newPending...)
-	heap.Init(&ss.pending)
+	if ss.stateful {
+		if ss.pendingByKeys == nil {
+			ss.pendingByKeys = map[string]elementHeap{}
+		}
+		for _, e := range newPending {
+			if len(e.keyBytes) == 0 {
+				panic(fmt.Sprintf("zero length key: %v %v", ss.ID, ss.inputID))
+			}
+			h := ss.pendingByKeys[string(e.keyBytes)]
+			h.Push(e)
+			ss.pendingByKeys[string(e.keyBytes)] = h // (Is this necessary, with the way the heap interface works over a slice?)
+		}
+	} else {
+		ss.pending = append(ss.pending, newPending...)
+		heap.Init(&ss.pending)
+	}
 }
 
 // AddPendingSide adds elements to be consumed as side inputs.
@@ -647,10 +779,16 @@ func (ss *stageState) OutputWatermark() mtime.Time {
 	return ss.output
 }
 
+// TODO: Move to better place for configuration
+var (
+	OneKeyPerBundle  bool // OneKeyPerBundle sets if a bundle is restricted to a single key.
+	OneElementPerKey bool // OneElementPerKey sets if a key in a bundle is restricted to one element.
+)
+
 // startBundle initializes a bundle with elements if possible.
 // A bundle only starts if there are elements at all, and if it's
 // an aggregation stage, if the windowing stratgy allows it.
-func (ss *stageState) startBundle(watermark mtime.Time, genBundID func() string) (string, bool) {
+func (ss *stageState) startBundle(watermark mtime.Time, genBundID func() string) (string, bool, bool) {
 	defer func() {
 		if e := recover(); e != nil {
 			panic(fmt.Sprintf("generating bundle for stage %v at %v panicked\n%v", ss.ID, watermark, e))
@@ -669,21 +807,73 @@ func (ss *stageState) startBundle(watermark mtime.Time, genBundID func() string)
 	}
 	ss.pending = notYet
 	heap.Init(&ss.pending)
+	if ss.inprogressKeys == nil {
+		ss.inprogressKeys = set[string]{}
+	}
+	minTs := mtime.MaxTimestamp
+	// TODO: Allow configurable limit of keys per bundle, and elements per key to improve parallelism.
+	// TODO: when we do, we need to ensure that the stage remains schedualable for bundle execution, for remaining pending elements and keys.
+	// With the greedy approach, we don't need to since "new data" triggers a refresh, and so should completing processing of a bundle.
+	newKeys := set[string]{}
+	stillSchedulable := true
+
+keysPerBundle:
+	for k, h := range ss.pendingByKeys {
+		if ss.inprogressKeys.present(k) {
+			continue
+		}
+		newKeys.insert(k)
+		// Track the min-timestamp for later watermark handling.
+		if h[0].timestamp < minTs {
+			minTs = h[0].timestamp
+		}
+
+		if OneElementPerKey {
+			hp := &h
+			toProcess = append(toProcess, heap.Pop(hp).(element))
+			if hp.Len() == 0 {
+				// Once we've taken all the elements for a key,
+				// we must delete them from pending as well.
+				delete(ss.pendingByKeys, k)
+			} else {
+				ss.pendingByKeys[k] = *hp
+			}
+		} else {
+			toProcess = append(toProcess, h...)
+			delete(ss.pendingByKeys, k)
+		}
+		if OneKeyPerBundle {
+			break keysPerBundle
+		}
+	}
+	if len(ss.pendingByKeys) == 0 {
+		stillSchedulable = false
+	}
 
 	if len(toProcess) == 0 {
-		return "", false
+		return "", false, false
+	}
+
+	if toProcess[0].timestamp < minTs {
+		// Catch the ordinary case.
+		minTs = toProcess[0].timestamp
 	}
-	// Is THIS is where basic splits should happen/per element processing?
+
 	es := elements{
 		es:           toProcess,
-		minTimestamp: toProcess[0].timestamp,
+		minTimestamp: minTs,
 	}
 	if ss.inprogress == nil {
 		ss.inprogress = make(map[string]elements)
 	}
+	if ss.inprogressKeysByBundle == nil {
+		ss.inprogressKeysByBundle = make(map[string]set[string])
+	}
 	bundID := genBundID()
 	ss.inprogress[bundID] = es
-	return bundID, true
+	ss.inprogressKeysByBundle[bundID] = newKeys
+	ss.inprogressKeys.merge(newKeys)
+	return bundID, true, stillSchedulable
 }
 
 func (ss *stageState) splitBundle(rb RunBundle, firstResidual int) {
@@ -713,6 +903,12 @@ func (ss *stageState) minPendingTimestamp() mtime.Time {
 	if len(ss.pending) != 0 {
 		minPending = ss.pending[0].timestamp
 	}
+	if len(ss.pendingByKeys) != 0 {
+		// TODO(lostluck): Can we figure out how to avoid checking every key on every watermark refresh?
+		for _, h := range ss.pendingByKeys {
+			minPending = mtime.Min(minPending, h[0].timestamp)
+		}
+	}
 	for _, es := range ss.inprogress {
 		minPending = mtime.Min(minPending, es.minTimestamp)
 	}
@@ -785,6 +981,14 @@ func (ss *stageState) updateWatermarks(minPending, minStateHold mtime.Time, em *
 				}
 			}
 		}
+		for _, wins := range ss.state {
+			for win := range wins {
+				// Clear out anything we've already used.
+				if win.MaxTimestamp() < newOut {
+					delete(wins, win)
+				}
+			}
+		}
 	}
 	return refreshes
 }
diff --git a/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go b/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go
new file mode 100644
index 000000000000..af41e089a2e9
--- /dev/null
+++ b/sdks/go/pkg/beam/runners/prism/internal/engine/engine_test.go
@@ -0,0 +1,159 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package engine_test ensures coverage of the element manager via pipeline actuation.
+package engine_test
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/util/reflectx"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/options/jobopts"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/engine"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/jobservices"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/universal"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest"
+	"github.com/apache/beam/sdks/v2/go/test/integration/primitives"
+)
+
+func init() {
+	// Not actually being used, but explicitly registering
+	// will avoid accidentally using a different runner for
+	// the tests if I change things later.
+	beam.RegisterRunner("testlocal", execute)
+}
+
+func TestMain(m *testing.M) {
+	ptest.MainWithDefault(m, "testlocal")
+}
+
+func initRunner(t testing.TB) {
+	t.Helper()
+	if *jobopts.Endpoint == "" {
+		s := jobservices.NewServer(0, internal.RunPipeline)
+		*jobopts.Endpoint = s.Endpoint()
+		go s.Serve()
+		t.Cleanup(func() {
+			*jobopts.Endpoint = ""
+			s.Stop()
+		})
+	}
+	if !jobopts.IsLoopback() {
+		*jobopts.EnvironmentType = "loopback"
+	}
+	// Since we force loopback, avoid cross-compilation.
+	f, err := os.CreateTemp("", "dummy")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { os.Remove(f.Name()) })
+	*jobopts.WorkerBinary = f.Name()
+}
+
+func execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) {
+	return universal.Execute(ctx, p)
+}
+
+func executeWithT(ctx context.Context, t testing.TB, p *beam.Pipeline) (beam.PipelineResult, error) {
+	t.Helper()
+	t.Log("startingTest - ", t.Name())
+	s1 := rand.NewSource(time.Now().UnixNano())
+	r1 := rand.New(s1)
+	*jobopts.JobName = fmt.Sprintf("%v-%v", strings.ToLower(t.Name()), r1.Intn(1000))
+	return execute(ctx, p)
+}
+
+func initTestName(fn any) string {
+	name := reflectx.FunctionName(fn)
+	n := strings.LastIndex(name, "/")
+	return name[n+1:]
+}
+
+func TestStateAPI(t *testing.T) {
+	initRunner(t)
+
+	tests := []struct {
+		pipeline func(s beam.Scope)
+	}{
+		{pipeline: primitives.BagStateParDo},
+		{pipeline: primitives.BagStateParDoClear},
+		{pipeline: primitives.CombiningStateParDo},
+		{pipeline: primitives.ValueStateParDo},
+		{pipeline: primitives.ValueStateParDoClear},
+		{pipeline: primitives.ValueStateParDoWindowed},
+		{pipeline: primitives.MapStateParDo},
+		{pipeline: primitives.MapStateParDoClear},
+		{pipeline: primitives.SetStateParDo},
+		{pipeline: primitives.SetStateParDoClear},
+	}
+
+	configs := []struct {
+		name                              string
+		OneElementPerKey, OneKeyPerBundle bool
+	}{
+		{"Greedy", false, false},
+		{"AllElementsPerKey", false, true},
+		{"OneElementPerKey", true, false},
+		{"OneElementPerBundle", true, true},
+	}
+	for _, config := range configs {
+		for _, test := range tests {
+			t.Run(initTestName(test.pipeline)+"_"+config.name, func(t *testing.T) {
+				t.Cleanup(func() {
+					engine.OneElementPerKey = false
+					engine.OneKeyPerBundle = false
+				})
+				engine.OneElementPerKey = config.OneElementPerKey
+				engine.OneKeyPerBundle = config.OneKeyPerBundle
+				p, s := beam.NewPipelineWithRoot()
+				test.pipeline(s)
+				_, err := executeWithT(context.Background(), t, p)
+				if err != nil {
+					t.Fatalf("pipeline failed, but feature should be implemented in Prism: %v", err)
+				}
+			})
+		}
+	}
+}
+
+func TestElementManagerCoverage(t *testing.T) {
+	initRunner(t)
+
+	tests := []struct {
+		pipeline func(s beam.Scope)
+	}{
+		{pipeline: primitives.Checkpoints}, // (Doesn't run long enough to split.)
+		{pipeline: primitives.WindowSums_Lifted},
+	}
+
+	for _, test := range tests {
+		t.Run(initTestName(test.pipeline), func(t *testing.T) {
+			p, s := beam.NewPipelineWithRoot()
+			test.pipeline(s)
+			_, err := executeWithT(context.Background(), t, p)
+			if err != nil {
+				t.Fatalf("pipeline failed, but feature should be implemented in Prism: %v", err)
+			}
+		})
+	}
+}
diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute.go b/sdks/go/pkg/beam/runners/prism/internal/execute.go
index 895122383857..b8bc68dcd1b7 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/execute.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/execute.go
@@ -179,11 +179,16 @@ func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservic
 			ed := collectionPullDecoder(col.GetCoderId(), coders, comps)
 			wDec, wEnc := getWindowValueCoders(comps, col, coders)
 
+			var kd func(io.Reader) []byte
+			if kcid, ok := extractKVCoderID(col.GetCoderId(), coders); ok {
+				kd = collectionPullDecoder(kcid, coders, comps)
+			}
 			stage.OutputsToCoders[onlyOut] = engine.PColInfo{
 				GlobalID: onlyOut,
 				WDec:     wDec,
 				WEnc:     wEnc,
 				EDec:     ed,
+				KeyDec:   kd,
 			}
 
 			// There's either 0, 1 or many inputs, but they should be all the same
@@ -208,11 +213,17 @@ func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservic
 					col := comps.GetPcollections()[global]
 					ed := collectionPullDecoder(col.GetCoderId(), coders, comps)
 					wDec, wEnc := getWindowValueCoders(comps, col, coders)
+
+					var kd func(io.Reader) []byte
+					if kcid, ok := extractKVCoderID(col.GetCoderId(), coders); ok {
+						kd = collectionPullDecoder(kcid, coders, comps)
+					}
 					stage.inputInfo = engine.PColInfo{
 						GlobalID: global,
 						WDec:     wDec,
 						WEnc:     wEnc,
 						EDec:     ed,
+						KeyDec:   kd,
 					}
 				}
 				em.StageAggregates(stage.ID)
@@ -234,6 +245,9 @@ func executePipeline(ctx context.Context, wks map[string]*worker.W, j *jobservic
 			outputs := maps.Keys(stage.OutputsToCoders)
 			sort.Strings(outputs)
 			em.AddStage(stage.ID, []string{stage.primaryInput}, outputs, stage.sideInputs)
+			if stage.stateful {
+				em.StageStateful(stage.ID)
+			}
 		default:
 			err := fmt.Errorf("unknown environment[%v]", t.GetEnvironmentId())
 			slog.Error("Execute", err)
@@ -286,6 +300,14 @@ func collectionPullDecoder(coldCId string, coders map[string]*pipepb.Coder, comp
 	return pullDecoder(coders[cID], coders)
 }
 
+func extractKVCoderID(coldCId string, coders map[string]*pipepb.Coder) (string, bool) {
+	c := coders[coldCId]
+	if c.GetSpec().GetUrn() == urns.CoderKV {
+		return c.GetComponentCoderIds()[0], true
+	}
+	return "", false
+}
+
 func getWindowValueCoders(comps *pipepb.Components, col *pipepb.PCollection, coders map[string]*pipepb.Coder) (exec.WindowDecoder, exec.WindowEncoder) {
 	ws := comps.GetWindowingStrategies()[col.GetWindowingStrategyId()]
 	wcID, err := lpUnknownCoders(ws.GetWindowCoderId(), coders, comps.GetCoders())
diff --git a/sdks/go/pkg/beam/runners/prism/internal/execute_test.go b/sdks/go/pkg/beam/runners/prism/internal/execute_test.go
index fe3da83c67e2..29fccaeb238e 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/execute_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/execute_test.go
@@ -37,6 +37,10 @@ import (
 	"github.com/apache/beam/sdks/v2/go/test/integration/primitives"
 )
 
+func TestMain(m *testing.M) {
+	ptest.MainWithDefault(m, "testlocal")
+}
+
 func initRunner(t testing.TB) {
 	t.Helper()
 	if *jobopts.Endpoint == "" {
@@ -585,10 +589,6 @@ func init() {
 // There's a doubling bug since we re-use the same pcollection IDs for the source & sink, and
 // don't do any re-writing.
 
-func TestMain(m *testing.M) {
-	ptest.MainWithDefault(m, "testlocal")
-}
-
 func init() {
 	// Basic Registration
 	// beam.RegisterFunction(identity)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go b/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go
index 45223c1b2bcb..38e7e9454df5 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/handlepardo.go
@@ -82,19 +82,26 @@ func (h *pardo) PrepareTransform(tid string, t *pipepb.PTransform, comps *pipepb
 		!pdo.RequestsFinalization &&
 		!pdo.RequiresStableInput &&
 		!pdo.RequiresTimeSortedInput &&
-		len(pdo.StateSpecs) == 0 &&
 		len(pdo.TimerFamilySpecs) == 0 &&
 		pdo.RestrictionCoderId == "" {
 		// Which inputs are Side inputs don't change the graph further,
 		// so they're not included here. Any nearly any ParDo can have them.
 
 		// At their simplest, we don't need to do anything special at pre-processing time, and simply pass through as normal.
+
+		// StatefulDoFns need to be marked as being roots.
+		var forcedRoots []string
+		if len(pdo.StateSpecs)+len(pdo.TimerFamilySpecs) > 0 {
+			forcedRoots = append(forcedRoots, tid)
+		}
+
 		return prepareResult{
 			SubbedComps: &pipepb.Components{
 				Transforms: map[string]*pipepb.PTransform{
 					tid: t,
 				},
 			},
+			ForcedRoots: forcedRoots,
 		}
 	}
 
diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go
index cd302a70fcc0..d6e906bee59f 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/job.go
@@ -42,7 +42,8 @@ import (
 )
 
 var supportedRequirements = map[string]struct{}{
-	urns.RequirementSplittableDoFn: {},
+	urns.RequirementSplittableDoFn:     {},
+	urns.RequirementStatefulProcessing: {},
 }
 
 // TODO, move back to main package, and key off of executor handlers?
diff --git a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
index 0fd7381e17f4..d3727b650860 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/jobservices/management.go
@@ -26,6 +26,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slog"
+	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
@@ -110,11 +111,10 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (*jo
 	// Inspect Transforms for unsupported features.
 	bypassedWindowingStrategies := map[string]bool{}
 	ts := job.Pipeline.GetComponents().GetTransforms()
-	for _, t := range ts {
+	for tid, t := range ts {
 		urn := t.GetSpec().GetUrn()
 		switch urn {
 		case urns.TransformImpulse,
-			urns.TransformParDo,
 			urns.TransformGBK,
 			urns.TransformFlatten,
 			urns.TransformCombinePerKey,
@@ -140,6 +140,22 @@ func (s *Server) Prepare(ctx context.Context, req *jobpb.PrepareJobRequest) (*jo
 				wsID := pcs[col].GetWindowingStrategyId()
 				bypassedWindowingStrategies[wsID] = true
 			}
+
+		case urns.TransformParDo:
+			var pardo pipepb.ParDoPayload
+			if err := proto.Unmarshal(t.GetSpec().GetPayload(), &pardo); err != nil {
+				return nil, fmt.Errorf("unable to unmarshal ParDoPayload for %v - %q: %w", tid, t.GetUniqueName(), err)
+			}
+
+			// Validate all the state features
+			for _, spec := range pardo.GetStateSpecs() {
+				check("StateSpec.Protocol.Urn", spec.GetProtocol().GetUrn(), urns.UserStateBag, urns.UserStateMultiMap)
+			}
+			// Validate all the timer features
+			for _, spec := range pardo.GetTimerFamilySpecs() {
+				check("TimerFamilySpecs.TimeDomain.Urn", spec.GetTimeDomain())
+			}
+
 		case "":
 			// Composites can often have no spec
 			if len(t.GetSubtransforms()) > 0 {
diff --git a/sdks/go/pkg/beam/runners/prism/internal/preprocess.go b/sdks/go/pkg/beam/runners/prism/internal/preprocess.go
index 494baa5b4a93..ea4cf2c99695 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/preprocess.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/preprocess.go
@@ -26,6 +26,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
 	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slog"
+	"google.golang.org/protobuf/proto"
 )
 
 // transformPreparer is an interface for handling different urns in the preprocessor
@@ -440,7 +441,18 @@ func finalizeStage(stg *stage, comps *pipepb.Components, pipelineFacts *fusionFa
 		inputs[pid] = true
 		for _, link := range plinks {
 			t := comps.GetTransforms()[link.Transform]
-			sis, _ := getSideInputs(t)
+
+			var sis map[string]*pipepb.SideInput
+			if t.GetSpec().GetUrn() == urns.TransformParDo {
+				pardo := &pipepb.ParDoPayload{}
+				if err := (proto.UnmarshalOptions{}).Unmarshal(t.GetSpec().GetPayload(), pardo); err != nil {
+					return fmt.Errorf("unable to decode ParDoPayload for %v", link.Transform)
+				}
+				if len(pardo.GetTimerFamilySpecs())+len(pardo.GetStateSpecs()) > 0 {
+					stg.stateful = true
+				}
+				sis = pardo.GetSideInputs()
+			}
 			if _, ok := sis[link.Local]; ok {
 				sideInputs = append(sideInputs, engine.LinkID{Transform: link.Transform, Global: link.Global, Local: link.Local})
 			} else {
diff --git a/sdks/go/pkg/beam/runners/prism/internal/stage.go b/sdks/go/pkg/beam/runners/prism/internal/stage.go
index 1ce102406381..17c2a67b95c6 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/stage.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/stage.go
@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/graph/mtime"
@@ -61,6 +62,7 @@ type stage struct {
 	sideInputs   []engine.LinkID // Non-parallel input PCollections and their consumers
 	internalCols []string        // PCollections that escape. Used for precise coder sending.
 	envID        string
+	stateful     bool
 
 	exe              transformExecuter
 	inputTransformID string
@@ -77,6 +79,7 @@ func (s *stage) Execute(ctx context.Context, j *jobservices.Job, wk *worker.W, c
 
 	var b *worker.B
 	inputData := em.InputForBundle(rb, s.inputInfo)
+	initialState := em.StateForBundle(rb)
 	var dataReady <-chan struct{}
 	switch s.envID {
 	case "": // Runner Transforms
@@ -102,8 +105,8 @@ func (s *stage) Execute(ctx context.Context, j *jobservices.Job, wk *worker.W, c
 
 			InputTransformID: s.inputTransformID,
 
-			// TODO Here's where we can split data for processing in multiple bundles.
-			InputData: inputData,
+			InputData:  inputData,
+			OutputData: initialState,
 
 			SinkToPCollection: s.SinkToPCollection,
 			OutputCount:       len(s.outputs),
@@ -203,7 +206,7 @@ progress:
 		}
 	}
 	// Tentative Data is ready, commit it to the main datastore.
-	slog.Debug("Execute: commiting data", "bundle", rb, slog.Any("outputsWithData", maps.Keys(b.OutputData.Raw)), slog.Any("outputs", maps.Keys(s.OutputsToCoders)))
+	slog.Debug("Execute: committing data", "bundle", rb, slog.Any("outputsWithData", maps.Keys(b.OutputData.Raw)), slog.Any("outputs", maps.Keys(s.OutputsToCoders)))
 
 	// Tally metrics immeadiately so they're available before
 	// pipeline termination.
@@ -300,6 +303,12 @@ func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, em *eng
 		}
 		sinkID := o.Transform + "_" + o.Local
 		ed := collectionPullDecoder(col.GetCoderId(), coders, comps)
+
+		var kd func(io.Reader) []byte
+		if kcid, ok := extractKVCoderID(col.GetCoderId(), coders); ok {
+			kd = collectionPullDecoder(kcid, coders, comps)
+		}
+
 		wDec, wEnc := getWindowValueCoders(comps, col, coders)
 		sink2Col[sinkID] = o.Global
 		col2Coders[o.Global] = engine.PColInfo{
@@ -307,6 +316,7 @@ func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, em *eng
 			WDec:     wDec,
 			WEnc:     wEnc,
 			EDec:     ed,
+			KeyDec:   kd,
 		}
 		transforms[sinkID] = sinkTransform(sinkID, portFor(wOutCid, wk), o.Global)
 	}
@@ -350,14 +360,20 @@ func buildDescriptor(stg *stage, comps *pipepb.Components, wk *worker.W, em *eng
 	if err != nil {
 		return fmt.Errorf("buildDescriptor: failed to handle coder on stage %v for primary input, pcol %q %v:\n%w\n%v", stg.ID, stg.primaryInput, prototext.Format(col), err, stg.transforms)
 	}
-
 	ed := collectionPullDecoder(col.GetCoderId(), coders, comps)
 	wDec, wEnc := getWindowValueCoders(comps, col, coders)
+
+	var kd func(io.Reader) []byte
+	if kcid, ok := extractKVCoderID(col.GetCoderId(), coders); ok {
+		kd = collectionPullDecoder(kcid, coders, comps)
+	}
+
 	inputInfo := engine.PColInfo{
 		GlobalID: stg.primaryInput,
 		WDec:     wDec,
 		WEnc:     wEnc,
 		EDec:     ed,
+		KeyDec:   kd,
 	}
 
 	stg.inputTransformID = stg.ID + "_source"
diff --git a/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go b/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go
index b8a04a7306b2..323773bd4cd6 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/unimplemented_test.go
@@ -27,7 +27,7 @@ import (
 
 // This file covers pipelines with features that aren't yet supported by Prism.
 
-func intTestName(fn any) string {
+func initTestName(fn any) string {
 	name := reflectx.FunctionName(fn)
 	n := strings.LastIndex(name, "/")
 	return name[n+1:]
@@ -68,23 +68,11 @@ func TestUnimplemented(t *testing.T) {
 		{pipeline: primitives.TriggerOrFinally},
 		{pipeline: primitives.TriggerRepeat},
 
-		// State API
-		{pipeline: primitives.BagStateParDo},
-		{pipeline: primitives.BagStateParDoClear},
-		{pipeline: primitives.MapStateParDo},
-		{pipeline: primitives.MapStateParDoClear},
-		{pipeline: primitives.SetStateParDo},
-		{pipeline: primitives.SetStateParDoClear},
-		{pipeline: primitives.CombiningStateParDo},
-		{pipeline: primitives.ValueStateParDo},
-		{pipeline: primitives.ValueStateParDoClear},
-		{pipeline: primitives.ValueStateParDoWindowed},
-
 		// TODO: Timers integration tests.
 	}
 
 	for _, test := range tests {
-		t.Run(intTestName(test.pipeline), func(t *testing.T) {
+		t.Run(initTestName(test.pipeline), func(t *testing.T) {
 			p, s := beam.NewPipelineWithRoot()
 			test.pipeline(s)
 			_, err := executeWithT(context.Background(), t, p)
@@ -113,7 +101,37 @@ func TestImplemented(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		t.Run(intTestName(test.pipeline), func(t *testing.T) {
+		t.Run(initTestName(test.pipeline), func(t *testing.T) {
+			p, s := beam.NewPipelineWithRoot()
+			test.pipeline(s)
+			_, err := executeWithT(context.Background(), t, p)
+			if err != nil {
+				t.Fatalf("pipeline failed, but feature should be implemented in Prism: %v", err)
+			}
+		})
+	}
+}
+
+func TestStateAPI(t *testing.T) {
+	initRunner(t)
+
+	tests := []struct {
+		pipeline func(s beam.Scope)
+	}{
+		{pipeline: primitives.BagStateParDo},
+		{pipeline: primitives.BagStateParDoClear},
+		{pipeline: primitives.CombiningStateParDo},
+		{pipeline: primitives.ValueStateParDo},
+		{pipeline: primitives.ValueStateParDoClear},
+		{pipeline: primitives.ValueStateParDoWindowed},
+		{pipeline: primitives.MapStateParDo},
+		{pipeline: primitives.MapStateParDoClear},
+		{pipeline: primitives.SetStateParDo},
+		{pipeline: primitives.SetStateParDoClear},
+	}
+
+	for _, test := range tests {
+		t.Run(initTestName(test.pipeline), func(t *testing.T) {
 			p, s := beam.NewPipelineWithRoot()
 			test.pipeline(s)
 			_, err := executeWithT(context.Background(), t, p)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go b/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go
index bf1e36656661..5312fd799c89 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/urns/urns.go
@@ -51,6 +51,7 @@ var (
 	reqUrn     = toUrn[pipepb.StandardRequirements_Enum]()
 	runProcUrn = toUrn[pipepb.StandardRunnerProtocols_Enum]()
 	envUrn     = toUrn[pipepb.StandardEnvironments_Environments]()
+	usUrn      = toUrn[pipepb.StandardUserStateTypes_Enum]()
 )
 
 var (
@@ -93,6 +94,10 @@ var (
 	SideInputIterable = siUrn(pipepb.StandardSideInputTypes_ITERABLE)
 	SideInputMultiMap = siUrn(pipepb.StandardSideInputTypes_MULTIMAP)
 
+	// UserState kinds
+	UserStateBag      = usUrn(pipepb.StandardUserStateTypes_BAG)
+	UserStateMultiMap = usUrn(pipepb.StandardUserStateTypes_MULTIMAP)
+
 	// WindowsFns
 	WindowFnGlobal  = quickUrn(pipepb.GlobalWindowsPayload_PROPERTIES)
 	WindowFnFixed   = quickUrn(pipepb.FixedWindowsPayload_PROPERTIES)
diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go b/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go
index 97250092940d..6ef3a81e6239 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/worker/bundle.go
@@ -42,11 +42,13 @@ type B struct {
 	InputTransformID string
 	InputData        [][]byte // Data specifically for this bundle.
 
-	// IterableSideInputData is a map from transformID, to inputID, to window, to data.
+	// IterableSideInputData is a map from transformID + inputID, to window, to data.
 	IterableSideInputData map[SideInputKey]map[typex.Window][][]byte
-	// MultiMapSideInputData is a map from transformID, to inputID, to window, to data key, to data values.
+	// MultiMapSideInputData is a map from transformID + inputID, to window, to data key, to data values.
 	MultiMapSideInputData map[SideInputKey]map[typex.Window]map[string][][]byte
 
+	// State lives in OutputData
+
 	// OutputCount is the number of data or timer outputs this bundle has.
 	// We need to see this many closed data channels before the bundle is complete.
 	OutputCount int
diff --git a/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go b/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go
index beee5e896ffc..2859dfe2356d 100644
--- a/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go
+++ b/sdks/go/pkg/beam/runners/prism/internal/worker/worker.go
@@ -36,6 +36,7 @@ import (
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/typex"
 	fnpb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/fnexecution_v1"
 	pipepb "github.com/apache/beam/sdks/v2/go/pkg/beam/model/pipeline_v1"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/engine"
 	"github.com/apache/beam/sdks/v2/go/pkg/beam/runners/prism/internal/urns"
 	"golang.org/x/exp/slog"
 	"google.golang.org/grpc"
@@ -412,21 +413,21 @@ func (wk *W) State(state fnpb.BeamFnState_StateServer) error {
 					panic(err)
 				}
 			}
+
+			// State requests are always for an active ProcessBundle instruction
+			wk.mu.Lock()
+			b, ok := wk.activeInstructions[req.GetInstructionId()].(*B)
+			wk.mu.Unlock()
+			if !ok {
+				slog.Warn("state request after bundle inactive", "instruction", req.GetInstructionId(), "worker", wk)
+				continue
+			}
 			switch req.GetRequest().(type) {
 			case *fnpb.StateRequest_Get:
 				// TODO: move data handling to be pcollection based.
 
-				// State requests are always for an active ProcessBundle instruction
-				wk.mu.Lock()
-				b, ok := wk.activeInstructions[req.GetInstructionId()].(*B)
-				wk.mu.Unlock()
-				if !ok {
-					slog.Warn("state request after bundle inactive", "instruction", req.GetInstructionId(), "worker", wk)
-					continue
-				}
 				key := req.GetStateKey()
 				slog.Debug("StateRequest_Get", prototext.Format(req), "bundle", b)
-
 				var data [][]byte
 				switch key.GetType().(type) {
 				case *fnpb.StateKey_IterableSideInput_:
@@ -442,11 +443,13 @@ func (wk *W) State(state fnpb.BeamFnState_StateServer) error {
 						}
 					}
 					winMap := b.IterableSideInputData[SideInputKey{TransformID: ikey.GetTransformId(), Local: ikey.GetSideInputId()}]
+
 					var wins []typex.Window
 					for w := range winMap {
 						wins = append(wins, w)
 					}
 					slog.Debug(fmt.Sprintf("side input[%v][%v] I Key: %v Windows: %v", req.GetId(), req.GetInstructionId(), w, wins))
+
 					data = winMap[w]
 
 				case *fnpb.StateKey_MultimapSideInput_:
@@ -458,37 +461,81 @@ func (wk *W) State(state fnpb.BeamFnState_StateServer) error {
 					} else {
 						w, err = exec.MakeWindowDecoder(coder.NewIntervalWindow()).DecodeSingle(bytes.NewBuffer(wKey))
 						if err != nil {
-							panic(fmt.Sprintf("error decoding iterable side input window key %v: %v", wKey, err))
+							panic(fmt.Sprintf("error decoding multimap side input window key %v: %v", wKey, err))
 						}
 					}
 					dKey := mmkey.GetKey()
 					winMap := b.MultiMapSideInputData[SideInputKey{TransformID: mmkey.GetTransformId(), Local: mmkey.GetSideInputId()}]
-					var wins []typex.Window
-					for w := range winMap {
-						wins = append(wins, w)
-					}
-					slog.Debug(fmt.Sprintf("side input[%v][%v] MM Key: %v Windows: %v", req.GetId(), req.GetInstructionId(), w, wins))
+
+					slog.Debug(fmt.Sprintf("side input[%v][%v] MultiMap Window: %v", req.GetId(), req.GetInstructionId(), w))
 
 					data = winMap[w][string(dKey)]
 
+				case *fnpb.StateKey_BagUserState_:
+					bagkey := key.GetBagUserState()
+					data = b.OutputData.GetBagState(engine.LinkID{Transform: bagkey.GetTransformId(), Local: bagkey.GetUserStateId()}, bagkey.GetWindow(), bagkey.GetKey())
+				case *fnpb.StateKey_MultimapUserState_:
+					mmkey := key.GetMultimapUserState()
+					data = b.OutputData.GetMultimapState(engine.LinkID{Transform: mmkey.GetTransformId(), Local: mmkey.GetUserStateId()}, mmkey.GetWindow(), mmkey.GetKey(), mmkey.GetMapKey())
+				case *fnpb.StateKey_MultimapKeysUserState_:
+					mmkey := key.GetMultimapKeysUserState()
+					data = b.OutputData.GetMultimapKeysState(engine.LinkID{Transform: mmkey.GetTransformId(), Local: mmkey.GetUserStateId()}, mmkey.GetWindow(), mmkey.GetKey())
 				default:
-					panic(fmt.Sprintf("unsupported StateKey Access type: %T: %v", key.GetType(), prototext.Format(key)))
+					panic(fmt.Sprintf("unsupported StateKey Get type: %T: %v", key.GetType(), prototext.Format(key)))
 				}
 
 				// Encode the runner iterable (no length, just consecutive elements), and send it out.
 				// This is also where we can handle things like State Backed Iterables.
-				var buf bytes.Buffer
-				for _, value := range data {
-					buf.Write(value)
-				}
 				responses <- &fnpb.StateResponse{
 					Id: req.GetId(),
 					Response: &fnpb.StateResponse_Get{
 						Get: &fnpb.StateGetResponse{
-							Data: buf.Bytes(),
+							Data: bytes.Join(data, []byte{}),
 						},
 					},
 				}
+
+			case *fnpb.StateRequest_Append:
+				key := req.GetStateKey()
+				switch key.GetType().(type) {
+				case *fnpb.StateKey_BagUserState_:
+					bagkey := key.GetBagUserState()
+					b.OutputData.AppendBagState(engine.LinkID{Transform: bagkey.GetTransformId(), Local: bagkey.GetUserStateId()}, bagkey.GetWindow(), bagkey.GetKey(), req.GetAppend().GetData())
+				case *fnpb.StateKey_MultimapUserState_:
+					mmkey := key.GetMultimapUserState()
+					b.OutputData.AppendMultimapState(engine.LinkID{Transform: mmkey.GetTransformId(), Local: mmkey.GetUserStateId()}, mmkey.GetWindow(), mmkey.GetKey(), mmkey.GetMapKey(), req.GetAppend().GetData())
+				default:
+					panic(fmt.Sprintf("unsupported StateKey Append type: %T: %v", key.GetType(), prototext.Format(key)))
+				}
+				responses <- &fnpb.StateResponse{
+					Id: req.GetId(),
+					Response: &fnpb.StateResponse_Append{
+						Append: &fnpb.StateAppendResponse{},
+					},
+				}
+
+			case *fnpb.StateRequest_Clear:
+				key := req.GetStateKey()
+				switch key.GetType().(type) {
+				case *fnpb.StateKey_BagUserState_:
+					bagkey := key.GetBagUserState()
+					b.OutputData.ClearBagState(engine.LinkID{Transform: bagkey.GetTransformId(), Local: bagkey.GetUserStateId()}, bagkey.GetWindow(), bagkey.GetKey())
+				case *fnpb.StateKey_MultimapUserState_:
+					mmkey := key.GetMultimapUserState()
+					b.OutputData.ClearMultimapState(engine.LinkID{Transform: mmkey.GetTransformId(), Local: mmkey.GetUserStateId()}, mmkey.GetWindow(), mmkey.GetKey(), mmkey.GetMapKey())
+				case *fnpb.StateKey_MultimapKeysUserState_:
+					mmkey := key.GetMultimapUserState()
+					b.OutputData.ClearMultimapKeysState(engine.LinkID{Transform: mmkey.GetTransformId(), Local: mmkey.GetUserStateId()}, mmkey.GetWindow(), mmkey.GetKey())
+				default:
+					panic(fmt.Sprintf("unsupported StateKey Clear type: %T: %v", key.GetType(), prototext.Format(key)))
+				}
+				responses <- &fnpb.StateResponse{
+					Id: req.GetId(),
+					Response: &fnpb.StateResponse_Clear{
+						Clear: &fnpb.StateClearResponse{},
+					},
+				}
+
 			default:
 				panic(fmt.Sprintf("unsupported StateRequest kind %T: %v", req.GetRequest(), prototext.Format(req)))
 			}
diff --git a/sdks/go/test/integration/integration.go b/sdks/go/test/integration/integration.go
index 5b7473fb561a..160af1ab741f 100644
--- a/sdks/go/test/integration/integration.go
+++ b/sdks/go/test/integration/integration.go
@@ -103,6 +103,7 @@ var directFilters = []string{
 	"TestMapStateClear",
 	"TestSetState",
 	"TestSetStateClear",
+	"TestTimers.*", // no timer support for the go direct runner.
 }
 
 var portableFilters = []string{
@@ -125,17 +126,14 @@ var portableFilters = []string{
 	"TestFhirIO.*",
 	// OOMs currently only lead to heap dumps on Dataflow runner
 	"TestOomParDo",
-	// The portable runner does not support user state.
-	"TestValueState",
-	"TestValueStateWindowed",
-	"TestValueStateClear",
-	"TestBagState",
-	"TestBagStateClear",
-	"TestCombiningState",
+	// The portable runner does not support user map states.
 	"TestMapState",
 	"TestMapStateClear",
 	"TestSetState",
 	"TestSetStateClear",
+
+	// The portable runner does not uniquify timers. (data elements re-fired)
+	"TestTimers.*",
 }
 
 var prismFilters = []string{
@@ -158,17 +156,9 @@ var prismFilters = []string{
 	"TestFhirIO.*",
 	// OOMs currently only lead to heap dumps on Dataflow runner
 	"TestOomParDo",
-	// The prism runner does not support user state.
-	"TestValueState",
-	"TestValueStateWindowed",
-	"TestValueStateClear",
-	"TestBagState",
-	"TestBagStateClear",
-	"TestCombiningState",
-	"TestMapState",
-	"TestMapStateClear",
-	"TestSetState",
-	"TestSetStateClear",
+
+	// The prism runner does not support timers https://github.com/apache/beam/issues/29772.
+	"TestTimers.*",
 }
 
 var flinkFilters = []string{
@@ -192,6 +182,8 @@ var flinkFilters = []string{
 	"TestMapStateClear",
 	"TestSetStateClear",
 	"TestSetState",
+
+	"TestTimers_EventTime_Unbounded", // (failure when comparing on side inputs (NPE on window lookup))
 }
 
 var samzaFilters = []string{
@@ -230,6 +222,9 @@ var samzaFilters = []string{
 	"TestSetStateClear",
 	// TODO(https://github.com/apache/beam/issues/26126): Java runner issue (AcitveBundle has no regsitered handler)
 	"TestDebeziumIO_BasicRead",
+
+	// Samza does not support state.
+	"TestTimers.*",
 }
 
 var sparkFilters = []string{
@@ -261,6 +256,8 @@ var sparkFilters = []string{
 	"TestMapStateClear",
 	"TestSetStateClear",
 	"TestSetState",
+
+	"TestTimers_EventTime_Unbounded", // Side inputs in executable stage not supported.
 }
 
 var dataflowFilters = []string{
diff --git a/sdks/go/test/integration/primitives/state.go b/sdks/go/test/integration/primitives/state.go
index 5f105597ba37..acf1bf8fa665 100644
--- a/sdks/go/test/integration/primitives/state.go
+++ b/sdks/go/test/integration/primitives/state.go
@@ -39,6 +39,7 @@ func init() {
 	register.DoFn3x1[state.Provider, string, int, string](&mapStateClearFn{})
 	register.DoFn3x1[state.Provider, string, int, string](&setStateFn{})
 	register.DoFn3x1[state.Provider, string, int, string](&setStateClearFn{})
+	register.Function2x0(pairWithOne)
 	register.Emitter2[string, int]()
 	register.Combiner1[int](&combine1{})
 	register.Combiner2[string, int](&combine2{})
@@ -78,12 +79,14 @@ func (f *valueStateFn) ProcessElement(s state.Provider, w string, c int) string
 	return fmt.Sprintf("%s: %v, %s", w, i, j)
 }
 
+func pairWithOne(w string, emit func(string, int)) {
+	emit(w, 1)
+}
+
 // ValueStateParDo tests a DoFn that uses value state.
 func ValueStateParDo(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &valueStateFn{}, keyed)
 	passert.Equals(s, counts, "apple: 1, I", "pear: 1, I", "peach: 1, I", "apple: 2, II", "apple: 3, III", "pear: 2, II")
 }
@@ -124,9 +127,7 @@ func (f *valueStateClearFn) ProcessElement(s state.Provider, w string, c int) st
 // ValueStateParDoClear tests that a DoFn that uses value state can be cleared.
 func ValueStateParDoClear(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear", "pear", "apple")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &valueStateClearFn{State1: state.MakeValueState[int]("key1")}, keyed)
 	passert.Equals(s, counts, "apple: 0,false", "pear: 0,false", "peach: 0,false", "apple: 1,true", "apple: 0,false", "pear: 1,true", "pear: 0,false", "apple: 1,true")
 }
@@ -170,9 +171,7 @@ func (f *bagStateFn) ProcessElement(s state.Provider, w string, c int) string {
 // BagStateParDo tests a DoFn that uses bag state.
 func BagStateParDo(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &bagStateFn{}, keyed)
 	passert.Equals(s, counts, "apple: 0, ", "pear: 0, ", "peach: 0, ", "apple: 1, I", "apple: 2, I,I", "pear: 1, I")
 }
@@ -207,9 +206,7 @@ func (f *bagStateClearFn) ProcessElement(s state.Provider, w string, c int) stri
 // BagStateParDoClear tests a DoFn that uses bag state.
 func BagStateParDoClear(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "apple", "apple", "pear", "apple", "apple", "pear", "pear", "pear", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &bagStateClearFn{State1: state.MakeBagState[int]("key1")}, keyed)
 	passert.Equals(s, counts, "apple: 0", "pear: 0", "apple: 1", "apple: 2", "pear: 1", "apple: 3", "apple: 0", "pear: 2", "pear: 3", "pear: 0", "apple: 1", "pear: 1")
 }
@@ -312,16 +309,20 @@ func (f *combiningStateFn) ProcessElement(s state.Provider, w string, c int) str
 	return fmt.Sprintf("%s: %v %v %v %v %v", w, i, i1, i2, i3, i4)
 }
 
+func init() {
+	register.Function2x1(sumInt)
+}
+
+func sumInt(a, b int) int {
+	return a + b
+}
+
 // CombiningStateParDo tests a DoFn that uses value state.
 func CombiningStateParDo(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &combiningStateFn{
-		State0: state.MakeCombiningState[int, int, int]("key0", func(a, b int) int {
-			return a + b
-		}),
+		State0: state.MakeCombiningState[int, int, int]("key0", sumInt),
 		State1: state.Combining[int, int, int](state.MakeCombiningState[int, int, int]("key1", &combine1{})),
 		State2: state.Combining[string, string, int](state.MakeCombiningState[string, string, int]("key2", &combine2{})),
 		State3: state.Combining[string, string, int](state.MakeCombiningState[string, string, int]("key3", &combine3{})),
@@ -369,9 +370,7 @@ func (f *mapStateFn) ProcessElement(s state.Provider, w string, c int) string {
 // MapStateParDo tests a DoFn that uses value state.
 func MapStateParDo(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &mapStateFn{State1: state.MakeMapState[string, int]("key1")}, keyed)
 	passert.Equals(s, counts, "apple: 1, keys: [apple apple1]", "pear: 1, keys: [pear pear1]", "peach: 1, keys: [peach peach1]", "apple: 2, keys: [apple apple1 apple2]", "apple: 3, keys: [apple apple1 apple2 apple3]", "pear: 2, keys: [pear pear1 pear2]")
 }
@@ -425,9 +424,7 @@ func (f *mapStateClearFn) ProcessElement(s state.Provider, w string, c int) stri
 // MapStateParDoClear tests clearing and removing from a DoFn that uses map state.
 func MapStateParDoClear(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &mapStateClearFn{State1: state.MakeMapState[string, int]("key1")}, keyed)
 	passert.Equals(s, counts, "apple: [apple]", "pear: [pear]", "peach: [peach]", "apple: [apple1 apple2 apple3]", "apple: []", "pear: [pear1 pear2 pear3]")
 }
@@ -465,9 +462,7 @@ func (f *setStateFn) ProcessElement(s state.Provider, w string, c int) string {
 // SetStateParDo tests a DoFn that uses set state.
 func SetStateParDo(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &setStateFn{State1: state.MakeSetState[string]("key1")}, keyed)
 	passert.Equals(s, counts, "apple: false, keys: [apple]", "pear: false, keys: [pear]", "peach: false, keys: [peach]", "apple: true, keys: [apple apple1]", "apple: true, keys: [apple apple1]", "pear: true, keys: [pear pear1]")
 }
@@ -521,9 +516,7 @@ func (f *setStateClearFn) ProcessElement(s state.Provider, w string, c int) stri
 // SetStateParDoClear tests clearing and removing from a DoFn that uses set state.
 func SetStateParDoClear(s beam.Scope) {
 	in := beam.Create(s, "apple", "pear", "peach", "apple", "apple", "pear")
-	keyed := beam.ParDo(s, func(w string, emit func(string, int)) {
-		emit(w, 1)
-	}, in)
+	keyed := beam.ParDo(s, pairWithOne, in)
 	counts := beam.ParDo(s, &setStateClearFn{State1: state.MakeSetState[string]("key1")}, keyed)
 	passert.Equals(s, counts, "apple: [apple]", "pear: [pear]", "peach: [peach]", "apple: [apple1 apple2 apple3]", "apple: []", "pear: [pear1 pear2 pear3]")
 }
diff --git a/sdks/go/test/integration/primitives/timers.go b/sdks/go/test/integration/primitives/timers.go
new file mode 100644
index 000000000000..b4443296ecce
--- /dev/null
+++ b/sdks/go/test/integration/primitives/timers.go
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package primitives
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/state"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/core/timers"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/register"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert"
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/transforms/periodic"
+)
+
+// Based on https://github.com/apache/beam/blob/master/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
+
+func init() {
+	register.DoFn2x0[[]byte, func(string, int)](&inputFn[string, int]{})
+	register.DoFn6x0[beam.Window, state.Provider, timers.Provider, string, int, func(kv[string, int])](&eventTimeFn{})
+	register.Emitter2[string, int]()
+	register.Emitter1[kv[string, int]]()
+}
+
+type kv[K, V any] struct {
+	Key   K
+	Value V
+}
+
+func kvfn[K, V any](k K, v V) kv[K, V] {
+	return kv[K, V]{k, v}
+}
+
+type inputFn[K, V any] struct {
+	Inputs []kv[K, V]
+}
+
+func (fn *inputFn[K, V]) ProcessElement(_ []byte, emit func(K, V)) {
+	for _, in := range fn.Inputs {
+		emit(in.Key, in.Value)
+	}
+}
+
+type eventTimeFn struct {
+	Callback timers.EventTime
+	MyKey    state.Value[string]
+
+	Offset      int
+	TimerOutput int
+}
+
+func (fn *eventTimeFn) ProcessElement(w beam.Window, sp state.Provider, tp timers.Provider, key string, value int, emit func(kv[string, int])) {
+	fn.Callback.Set(tp, w.MaxTimestamp().ToTime())
+	fn.MyKey.Write(sp, key)
+	emit(kvfn(key, value+fn.Offset))
+}
+
+func (fn *eventTimeFn) OnTimer(ctx context.Context, ts beam.EventTime, sp state.Provider, tp timers.Provider, key string, timer timers.Context, emit func(kv[string, int])) {
+	switch timer.Family {
+	case fn.Callback.Family:
+		switch timer.Tag {
+		case "":
+			read, ok, err := fn.MyKey.Read(sp)
+			if err != nil {
+				panic(err)
+			}
+			if !ok {
+				panic("State must be set for key: " + key)
+			}
+			emit(kvfn(read, fn.TimerOutput))
+		default:
+			panic("unexpected timer tag: " + timer.Family + " tag:" + timer.Tag + " want: \"\", for key:" + key)
+		}
+	default:
+		if fn.Callback.Family != timer.Family || timer.Tag != "" {
+			panic("unexpected timer family: " + timer.Family + " tag:" + timer.Tag + " want: " + fn.Callback.Family + ", for key:" + key)
+		}
+	}
+}
+
+// TimersEventTime takes in an impulse transform and produces a pipeline construction
+// function that validates EventTime timers.
+//
+// The impulse is provided outside to swap between a bounded impulse, and
+// an unbounded one, because the Go SDK uses that to determine if a pipeline
+// is "streaming" or not. This matters at least for executions on Dataflow.
+//
+// The test produces some number of key value elements, with various event time offsets,
+// expecting that a single timer event is what fires. The test validates that all the
+// input elements, and the timer elements have been emitted.
+func timersEventTimePipelineBuilder(makeImp func(s beam.Scope) beam.PCollection) func(s beam.Scope) {
+	return func(s beam.Scope) {
+		var inputs, wantOutputs []kv[string, int]
+
+		offset := 5000
+		timerOutput := 4093
+
+		numKeys := 50
+		numDuplicateTimers := 15
+
+		for key := 0; key < numKeys; key++ {
+			k := strconv.Itoa(key)
+			wantOutputs = append(wantOutputs, kvfn(k, timerOutput))
+
+			for i := 0; i < numDuplicateTimers; i++ {
+				inputs = append(inputs, kvfn(k, i))
+				wantOutputs = append(wantOutputs, kvfn(k, i+offset))
+			}
+		}
+
+		imp := makeImp(s)
+
+		keyed := beam.ParDo(s, &inputFn[string, int]{
+			Inputs: inputs,
+		}, imp)
+		times := beam.ParDo(s, &eventTimeFn{
+			Offset:      offset,
+			TimerOutput: timerOutput,
+			Callback:    timers.InEventTime("Callback"),
+			MyKey:       state.MakeValueState[string]("MyKey"),
+		}, keyed)
+		passert.EqualsList(s, times, wantOutputs)
+	}
+}
+
+// TimersEventTimeBounded validates event time timers in a bounded pipeline.
+func TimersEventTimeBounded(s beam.Scope) {
+	timersEventTimePipelineBuilder(beam.Impulse)(s)
+}
+
+// TimersEventTimeUnbounded validates event time timers in an unbounded pipeline.
+func TimersEventTimeUnbounded(s beam.Scope) {
+	timersEventTimePipelineBuilder(func(s beam.Scope) beam.PCollection {
+		now := time.Now()
+		return periodic.Impulse(s, now, now.Add(10*time.Second), 0, false)
+	})(s)
+}
diff --git a/sdks/go/test/integration/primitives/timers_test.go b/sdks/go/test/integration/primitives/timers_test.go
new file mode 100644
index 000000000000..01f2706c6293
--- /dev/null
+++ b/sdks/go/test/integration/primitives/timers_test.go
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package primitives
+
+import (
+	"testing"
+
+	"github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest"
+	"github.com/apache/beam/sdks/v2/go/test/integration"
+)
+
+func TestTimers_EventTime_Bounded(t *testing.T) {
+	integration.CheckFilters(t)
+	ptest.BuildAndRun(t, TimersEventTimeBounded)
+}
+
+func TestTimers_EventTime_Unbounded(t *testing.T) {
+	integration.CheckFilters(t)
+	ptest.BuildAndRun(t, TimersEventTimeUnbounded)
+}
+
+// TODO(https://github.com/apache/beam/issues/29772): Add ProcessingTime Timer tests.
diff --git a/sdks/java/container/license_scripts/dep_urls_java.yaml b/sdks/java/container/license_scripts/dep_urls_java.yaml
index 7244bfcc6e2a..05849646aa8e 100644
--- a/sdks/java/container/license_scripts/dep_urls_java.yaml
+++ b/sdks/java/container/license_scripts/dep_urls_java.yaml
@@ -46,7 +46,7 @@ jaxen:
   '1.1.6':
     type: "3-Clause BSD"
 libraries-bom:
-  '26.26.0':
+  '26.28.0':
     license: "https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-java/master/LICENSE"
     type: "Apache License 2.0"
 paranamer:
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
index bd0215e1326e..50158f109cb0 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/Pipeline.java
@@ -22,6 +22,7 @@
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Iterables.transform;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
@@ -43,6 +44,9 @@
 import org.apache.beam.sdk.schemas.SchemaRegistry;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
 import org.apache.beam.sdk.transforms.resourcehints.ResourceHints;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.values.PBegin;
@@ -318,6 +322,7 @@ public PipelineResult run(PipelineOptions options) {
     LOG.debug("Running {} via {}", this, runner);
     try {
       validate(options);
+      validateErrorHandlers();
       return runner.run(this);
     } catch (UserCodeException e) {
       // This serves to replace the stack with one that ends here and
@@ -343,6 +348,13 @@ public SchemaRegistry getSchemaRegistry() {
     return schemaRegistry;
   }
 
+  public <OutputT extends POutput> BadRecordErrorHandler<OutputT> registerBadRecordErrorHandler(
+      PTransform<PCollection<BadRecord>, OutputT> sinkTransform) {
+    BadRecordErrorHandler<OutputT> errorHandler = new BadRecordErrorHandler<>(sinkTransform, this);
+    errorHandlers.add(errorHandler);
+    return errorHandler;
+  }
+
   /////////////////////////////////////////////////////////////////////////////
   // Below here are operations that aren't normally called by users.
 
@@ -511,6 +523,8 @@ public static <InputT extends PInput, OutputT extends POutput> OutputT applyTran
   private final Multimap<String, PTransform<?, ?>> instancePerName = ArrayListMultimap.create();
   private final PipelineOptions defaultOptions;
 
+  private final List<ErrorHandler<?, ?>> errorHandlers = new ArrayList<>();
+
   private Pipeline(TransformHierarchy transforms, PipelineOptions options) {
     this.transforms = transforms;
     this.defaultOptions = options;
@@ -715,4 +729,14 @@ public boolean apply(@Nonnull final Map.Entry<K, Collection<V>> input) {
       return input != null && input.getValue().size() == 1;
     }
   }
+
+  private void validateErrorHandlers() {
+    for (ErrorHandler<?, ?> errorHandler : errorHandlers) {
+      if (!errorHandler.isClosed()) {
+        throw new IllegalStateException(
+            "One or more ErrorHandlers aren't closed, and this pipeline"
+                + "cannot be run. See the ErrorHandler documentation for expected usage");
+      }
+    }
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java
index 76fc1a70b78c..0bc984877217 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileIO.java
@@ -61,6 +61,8 @@
 import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
 import org.apache.beam.sdk.transforms.display.DisplayData;
 import org.apache.beam.sdk.transforms.display.HasDisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -236,6 +238,27 @@
  * destination-dependent: every window/pane for every destination will use the same number of shards
  * specified via {@link Write#withNumShards} or {@link Write#withSharding}.
  *
+ * <h3>Handling Errors</h3>
+ *
+ * <p>When using dynamic destinations, or when using a formatting function to format a record for
+ * writing, it's possible for an individual record to be malformed, causing an exception. By
+ * default, these exceptions are propagated to the runner causing the bundle to fail. These are
+ * usually retried, though this depends on the runner. Alternately, these errors can be routed to
+ * another {@link PTransform} by using {@link Write#withBadRecordErrorHandler(ErrorHandler)}. The
+ * ErrorHandler is registered with the pipeline (see below). See {@link ErrorHandler} for more
+ * documentation. Of note, this error handling only handles errors related to specific records. It
+ * does not handle errors related to connectivity, authorization, etc. as those should be retried by
+ * the runner.
+ *
+ * <pre>{@code
+ * PCollection<> records = ...;
+ * PTransform<PCollection<BadRecord>,?> alternateSink = ...;
+ * try (BadRecordErrorHandler<?> handler = pipeline.registerBadRecordErrorHandler(alternateSink) {
+ *    records.apply("Write", FileIO.writeDynamic().otherConfigs()
+ *        .withBadRecordErrorHandler(handler));
+ * }
+ * }</pre>
+ *
  * <h3>Writing custom types to sinks</h3>
  *
  * <p>Normally, when writing a collection of a custom type using a {@link Sink} that takes a
@@ -1016,6 +1039,8 @@ public static FileNaming relativeFileNaming(
 
     abstract boolean getNoSpilling();
 
+    abstract @Nullable ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
     abstract Builder<DestinationT, UserT> toBuilder();
 
     @AutoValue.Builder
@@ -1062,6 +1087,9 @@ abstract Builder<DestinationT, UserT> setSharding(
 
       abstract Builder<DestinationT, UserT> setNoSpilling(boolean noSpilling);
 
+      abstract Builder<DestinationT, UserT> setBadRecordErrorHandler(
+          @Nullable ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
       abstract Write<DestinationT, UserT> build();
     }
 
@@ -1288,6 +1316,18 @@ public Write<DestinationT, UserT> withNoSpilling() {
       return toBuilder().setNoSpilling(true).build();
     }
 
+    /**
+     * Configures a new {@link Write} with an ErrorHandler. For configuring an ErrorHandler, see
+     * {@link ErrorHandler}. Whenever a record is formatted, or a lookup for a dynamic destination
+     * is performed, and that operation fails, the exception is passed to the error handler. This is
+     * intended to handle any errors related to the data of a record, but not any connectivity or IO
+     * errors related to the literal writing of a record.
+     */
+    public Write<DestinationT, UserT> withBadRecordErrorHandler(
+        ErrorHandler<BadRecord, ?> errorHandler) {
+      return toBuilder().setBadRecordErrorHandler(errorHandler).build();
+    }
+
     @VisibleForTesting
     Contextful<Fn<DestinationT, FileNaming>> resolveFileNamingFn() {
       if (getDynamic()) {
@@ -1391,6 +1431,9 @@ public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
       if (getNoSpilling()) {
         writeFiles = writeFiles.withNoSpilling();
       }
+      if (getBadRecordErrorHandler() != null) {
+        writeFiles = writeFiles.withBadRecordErrorHandler(getBadRecordErrorHandler());
+      }
       return input.apply(writeFiles);
     }
 
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
index eb78bd0e27d2..c58ce9ad8308 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/FileSystems.java
@@ -47,6 +47,7 @@
 import org.apache.beam.sdk.io.fs.MatchResult.Status;
 import org.apache.beam.sdk.io.fs.MoveOptions;
 import org.apache.beam.sdk.io.fs.MoveOptions.StandardMoveOptions;
+import org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions;
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.util.common.ReflectHelpers;
@@ -603,4 +604,20 @@ public static ResourceId matchNewResource(String singleResourceSpec, boolean isD
     return getFileSystemInternal(parseScheme(singleResourceSpec))
         .matchNewResource(singleResourceSpec, isDirectory);
   }
+
+  /**
+   * Returns a new {@link ResourceId} that represents the named directory resource.
+   *
+   * @param singleResourceSpec the root directory, for example "/abc"
+   * @param baseNames a list of named directory, for example ["d", "e", "f"]
+   * @return the ResourceId for the resolved directory. In same example as above, it corresponds to
+   *     "/abc/d/e/f".
+   */
+  public static ResourceId matchNewDirectory(String singleResourceSpec, String... baseNames) {
+    ResourceId currentDir = matchNewResource(singleResourceSpec, true);
+    for (String dir : baseNames) {
+      currentDir = currentDir.resolve(dir, StandardResolveOptions.RESOLVE_DIRECTORY);
+    }
+    return currentDir;
+  }
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
index 2c7a4fc5d4f5..96635a37fac1 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/TextIO.java
@@ -51,6 +51,8 @@
 import org.apache.beam.sdk.transforms.SerializableFunctions;
 import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
@@ -176,6 +178,10 @@
  *
  * <p>For backwards compatibility, {@link TextIO} also supports the legacy {@link
  * DynamicDestinations} interface for advanced features via {@link Write#to(DynamicDestinations)}.
+ *
+ * <p>Error handling for records that are malformed can be handled by using {@link
+ * TypedWrite#withBadRecordErrorHandler(ErrorHandler)}. See documentation in {@link FileIO} for
+ * details on usage
  */
 @SuppressWarnings({
   "nullness" // TODO(https://github.com/apache/beam/issues/20497)
@@ -708,6 +714,8 @@ public abstract static class TypedWrite<UserT, DestinationT>
      */
     abstract WritableByteChannelFactory getWritableByteChannelFactory();
 
+    abstract @Nullable ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
     abstract Builder<UserT, DestinationT> toBuilder();
 
     @AutoValue.Builder
@@ -754,6 +762,9 @@ abstract Builder<UserT, DestinationT> setNumShards(
       abstract Builder<UserT, DestinationT> setWritableByteChannelFactory(
           WritableByteChannelFactory writableByteChannelFactory);
 
+      abstract Builder<UserT, DestinationT> setBadRecordErrorHandler(
+          @Nullable ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
       abstract TypedWrite<UserT, DestinationT> build();
     }
 
@@ -993,6 +1004,12 @@ public TypedWrite<UserT, DestinationT> withNoSpilling() {
       return toBuilder().setNoSpilling(true).build();
     }
 
+    /** See {@link FileIO.Write#withBadRecordErrorHandler(ErrorHandler)} for details on usage. */
+    public TypedWrite<UserT, DestinationT> withBadRecordErrorHandler(
+        ErrorHandler<BadRecord, ?> errorHandler) {
+      return toBuilder().setBadRecordErrorHandler(errorHandler).build();
+    }
+
     /** Don't write any output files if the PCollection is empty. */
     public TypedWrite<UserT, DestinationT> skipIfEmpty() {
       return toBuilder().setSkipIfEmpty(true).build();
@@ -1083,6 +1100,9 @@ public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
       if (getNoSpilling()) {
         write = write.withNoSpilling();
       }
+      if (getBadRecordErrorHandler() != null) {
+        write = write.withBadRecordErrorHandler(getBadRecordErrorHandler());
+      }
       if (getSkipIfEmpty()) {
         write = write.withSkipIfEmpty();
       }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
index 91d6082eede4..7359141c5b87 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/WriteFiles.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.io;
 
+import static org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter.BAD_RECORD_TAG;
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
 
@@ -49,6 +50,7 @@
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.GroupIntoBatches;
@@ -62,6 +64,10 @@
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.DefaultErrorHandler;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
@@ -166,6 +172,8 @@ public static <UserT, DestinationT, OutputT> WriteFiles<UserT, DestinationT, Out
         .setMaxNumWritersPerBundle(DEFAULT_MAX_NUM_WRITERS_PER_BUNDLE)
         .setSideInputs(sink.getDynamicDestinations().getSideInputs())
         .setSkipIfEmpty(false)
+        .setBadRecordErrorHandler(new DefaultErrorHandler<>())
+        .setBadRecordRouter(BadRecordRouter.THROWING_ROUTER)
         .build();
   }
 
@@ -189,6 +197,10 @@ public static <UserT, DestinationT, OutputT> WriteFiles<UserT, DestinationT, Out
 
   public abstract @Nullable ShardingFunction<UserT, DestinationT> getShardingFunction();
 
+  public abstract ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
+  public abstract BadRecordRouter getBadRecordRouter();
+
   abstract Builder<UserT, DestinationT, OutputT> toBuilder();
 
   @AutoValue.Builder
@@ -215,6 +227,12 @@ abstract Builder<UserT, DestinationT, OutputT> setSideInputs(
     abstract Builder<UserT, DestinationT, OutputT> setShardingFunction(
         @Nullable ShardingFunction<UserT, DestinationT> shardingFunction);
 
+    abstract Builder<UserT, DestinationT, OutputT> setBadRecordErrorHandler(
+        ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
+    abstract Builder<UserT, DestinationT, OutputT> setBadRecordRouter(
+        BadRecordRouter badRecordRouter);
+
     abstract WriteFiles<UserT, DestinationT, OutputT> build();
   }
 
@@ -330,6 +348,15 @@ public WriteFiles<UserT, DestinationT, OutputT> withSkipIfEmpty() {
     return toBuilder().setSkipIfEmpty(true).build();
   }
 
+  /** See {@link FileIO.Write#withBadRecordErrorHandler(ErrorHandler)} for details on usage. */
+  public WriteFiles<UserT, DestinationT, OutputT> withBadRecordErrorHandler(
+      ErrorHandler<BadRecord, ?> errorHandler) {
+    return toBuilder()
+        .setBadRecordErrorHandler(errorHandler)
+        .setBadRecordRouter(BadRecordRouter.RECORDING_ROUTER)
+        .build();
+  }
+
   @Override
   public void validate(PipelineOptions options) {
     getSink().validate(options);
@@ -495,28 +522,39 @@ private WriteUnshardedBundlesToTempFiles(
 
     @Override
     public PCollection<FileResult<DestinationT>> expand(PCollection<UserT> input) {
-      if (getMaxNumWritersPerBundle() < 0) {
-        return input
-            .apply(
-                "WritedUnshardedBundles",
-                ParDo.of(new WriteUnshardedTempFilesFn(null, destinationCoder))
-                    .withSideInputs(getSideInputs()))
-            .setCoder(fileResultCoder);
-      }
       TupleTag<FileResult<DestinationT>> writtenRecordsTag = new TupleTag<>("writtenRecords");
       TupleTag<KV<ShardedKey<Integer>, UserT>> unwrittenRecordsTag =
           new TupleTag<>("unwrittenRecords");
+      Coder<UserT> inputCoder = input.getCoder();
+      if (getMaxNumWritersPerBundle() < 0) {
+        PCollectionTuple writeTuple =
+            input.apply(
+                "WritedUnshardedBundles",
+                ParDo.of(new WriteUnshardedTempFilesFn(null, destinationCoder, inputCoder))
+                    .withSideInputs(getSideInputs())
+                    .withOutputTags(
+                        writtenRecordsTag, TupleTagList.of(ImmutableList.of(BAD_RECORD_TAG))));
+        addErrorCollection(writeTuple);
+        return writeTuple.get(writtenRecordsTag).setCoder(fileResultCoder);
+      }
+
       PCollectionTuple writeTuple =
           input.apply(
               "WriteUnshardedBundles",
-              ParDo.of(new WriteUnshardedTempFilesFn(unwrittenRecordsTag, destinationCoder))
+              ParDo.of(
+                      new WriteUnshardedTempFilesFn(
+                          unwrittenRecordsTag, destinationCoder, inputCoder))
                   .withSideInputs(getSideInputs())
-                  .withOutputTags(writtenRecordsTag, TupleTagList.of(unwrittenRecordsTag)));
+                  .withOutputTags(
+                      writtenRecordsTag,
+                      TupleTagList.of(ImmutableList.of(unwrittenRecordsTag, BAD_RECORD_TAG))));
+      addErrorCollection(writeTuple);
+
       PCollection<FileResult<DestinationT>> writtenBundleFiles =
           writeTuple.get(writtenRecordsTag).setCoder(fileResultCoder);
       // Any "spilled" elements are written using WriteShardedBundles. Assign shard numbers in
       // finalize to stay consistent with what WriteWindowedBundles does.
-      PCollection<FileResult<DestinationT>> writtenSpilledFiles =
+      PCollectionTuple spilledWriteTuple =
           writeTuple
               .get(unwrittenRecordsTag)
               .setCoder(KvCoder.of(ShardedKeyCoder.of(VarIntCoder.of()), input.getCoder()))
@@ -529,7 +567,15 @@ public PCollection<FileResult<DestinationT>> expand(PCollection<UserT> input) {
               .apply("GroupUnwritten", GroupByKey.create())
               .apply(
                   "WriteUnwritten",
-                  ParDo.of(new WriteShardsIntoTempFilesFn()).withSideInputs(getSideInputs()))
+                  ParDo.of(new WriteShardsIntoTempFilesFn(input.getCoder()))
+                      .withSideInputs(getSideInputs())
+                      .withOutputTags(writtenRecordsTag, TupleTagList.of(BAD_RECORD_TAG)));
+
+      addErrorCollection(spilledWriteTuple);
+
+      PCollection<FileResult<DestinationT>> writtenSpilledFiles =
+          spilledWriteTuple
+              .get(writtenRecordsTag)
               .setCoder(fileResultCoder)
               .apply(
                   "DropShardNum",
@@ -556,6 +602,8 @@ private class WriteUnshardedTempFilesFn extends DoFn<UserT, FileResult<Destinati
     private final @Nullable TupleTag<KV<ShardedKey<Integer>, UserT>> unwrittenRecordsTag;
     private final Coder<DestinationT> destinationCoder;
 
+    private final Coder<UserT> inputCoder;
+
     // Initialized in startBundle()
     private @Nullable Map<WriterKey<DestinationT>, Writer<DestinationT, OutputT>> writers;
 
@@ -563,9 +611,11 @@ private class WriteUnshardedTempFilesFn extends DoFn<UserT, FileResult<Destinati
 
     WriteUnshardedTempFilesFn(
         @Nullable TupleTag<KV<ShardedKey<Integer>, UserT>> unwrittenRecordsTag,
-        Coder<DestinationT> destinationCoder) {
+        Coder<DestinationT> destinationCoder,
+        Coder<UserT> inputCoder) {
       this.unwrittenRecordsTag = unwrittenRecordsTag;
       this.destinationCoder = destinationCoder;
+      this.inputCoder = inputCoder;
     }
 
     @StartBundle
@@ -575,7 +625,9 @@ public void startBundle(StartBundleContext c) {
     }
 
     @ProcessElement
-    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
+    public void processElement(
+        ProcessContext c, BoundedWindow window, MultiOutputReceiver outputReceiver)
+        throws Exception {
       getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
       PaneInfo paneInfo = c.pane();
       // If we are doing windowed writes, we need to ensure that we have separate files for
@@ -583,7 +635,12 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
       // destinations go to different writers.
       // In the case of unwindowed writes, the window and the pane will always be the same, and
       // the map will only have a single element.
-      DestinationT destination = getDynamicDestinations().getDestination(c.element());
+      MaybeDestination<DestinationT> maybeDestination =
+          getDestinationWithErrorHandling(c.element(), outputReceiver, inputCoder);
+      if (!maybeDestination.isValid) {
+        return;
+      }
+      DestinationT destination = maybeDestination.destination;
       WriterKey<DestinationT> key = new WriterKey<>(window, c.pane(), destination);
       Writer<DestinationT, OutputT> writer = writers.get(key);
       if (writer == null) {
@@ -607,15 +664,22 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
           } else {
             spilledShardNum = (spilledShardNum + 1) % SPILLED_RECORD_SHARDING_FACTOR;
           }
-          c.output(
-              unwrittenRecordsTag,
-              KV.of(
-                  ShardedKey.of(hashDestination(destination, destinationCoder), spilledShardNum),
-                  c.element()));
+          outputReceiver
+              .get(unwrittenRecordsTag)
+              .output(
+                  KV.of(
+                      ShardedKey.of(
+                          hashDestination(destination, destinationCoder), spilledShardNum),
+                      c.element()));
           return;
         }
       }
-      writeOrClose(writer, getDynamicDestinations().formatRecord(c.element()));
+      OutputT formattedRecord =
+          formatRecordWithErrorHandling(c.element(), outputReceiver, inputCoder);
+      if (formattedRecord == null) {
+        return;
+      }
+      writeOrClose(writer, formattedRecord);
     }
 
     @FinishBundle
@@ -701,6 +765,56 @@ private static <DestinationT> int hashDestination(
         .asInt();
   }
 
+  private static class MaybeDestination<DestinationT> {
+    final DestinationT destination;
+    final boolean isValid;
+
+    MaybeDestination(DestinationT destination, boolean isValid) {
+      this.destination = destination;
+      this.isValid = isValid;
+    }
+  }
+  // Utility method to get the dynamic destination based on a record. Returns a MaybeDestination
+  // because some implementations of dynamic destinations return null, despite this being prohibited
+  // by the interface
+  private MaybeDestination<DestinationT> getDestinationWithErrorHandling(
+      UserT input, MultiOutputReceiver outputReceiver, Coder<UserT> inputCoder) throws Exception {
+    try {
+      return new MaybeDestination<>(getDynamicDestinations().getDestination(input), true);
+    } catch (Exception e) {
+      getBadRecordRouter()
+          .route(
+              outputReceiver, input, inputCoder, e, "Unable to get dynamic destination for record");
+      return new MaybeDestination<>(null, false);
+    }
+  }
+
+  // Utility method to format a record based on the dynamic destination. If the operation fails, and
+  // is output to the bad record router, this returns null
+  private @Nullable OutputT formatRecordWithErrorHandling(
+      UserT input, MultiOutputReceiver outputReceiver, Coder<UserT> inputCoder) throws Exception {
+    try {
+      return getDynamicDestinations().formatRecord(input);
+    } catch (Exception e) {
+      getBadRecordRouter()
+          .route(
+              outputReceiver,
+              input,
+              inputCoder,
+              e,
+              "Unable to format record for Dynamic Destination");
+      return null;
+    }
+  }
+
+  private void addErrorCollection(PCollectionTuple sourceTuple) {
+    getBadRecordErrorHandler()
+        .addErrorCollection(
+            sourceTuple
+                .get(BAD_RECORD_TAG)
+                .setCoder(BadRecord.getCoder(sourceTuple.getPipeline())));
+  }
+
   private class WriteShardedBundlesToTempFiles
       extends PTransform<PCollection<UserT>, PCollection<FileResult<DestinationT>>> {
     private final Coder<DestinationT> destinationCoder;
@@ -728,17 +842,32 @@ public PCollection<FileResult<DestinationT>> expand(PCollection<UserT> input) {
               ? new RandomShardingFunction(destinationCoder)
               : getShardingFunction();
 
-      return input
-          .apply(
+      TupleTag<KV<ShardedKey<Integer>, UserT>> shardedRecords = new TupleTag<>("shardedRecords");
+      TupleTag<FileResult<DestinationT>> writtenRecordsTag = new TupleTag<>("writtenRecords");
+
+      PCollectionTuple shardedFiles =
+          input.apply(
               "ApplyShardingKey",
-              ParDo.of(new ApplyShardingFunctionFn(shardingFunction, numShardsView))
-                  .withSideInputs(shardingSideInputs))
-          .setCoder(KvCoder.of(ShardedKeyCoder.of(VarIntCoder.of()), input.getCoder()))
-          .apply("GroupIntoShards", GroupByKey.create())
-          .apply(
-              "WriteShardsIntoTempFiles",
-              ParDo.of(new WriteShardsIntoTempFilesFn()).withSideInputs(getSideInputs()))
-          .setCoder(fileResultCoder);
+              ParDo.of(
+                      new ApplyShardingFunctionFn(
+                          shardingFunction, numShardsView, input.getCoder()))
+                  .withSideInputs(shardingSideInputs)
+                  .withOutputTags(shardedRecords, TupleTagList.of(BAD_RECORD_TAG)));
+      addErrorCollection(shardedFiles);
+
+      PCollectionTuple writtenFiles =
+          shardedFiles
+              .get(shardedRecords)
+              .setCoder(KvCoder.of(ShardedKeyCoder.of(VarIntCoder.of()), input.getCoder()))
+              .apply("GroupIntoShards", GroupByKey.create())
+              .apply(
+                  "WriteShardsIntoTempFiles",
+                  ParDo.of(new WriteShardsIntoTempFilesFn(input.getCoder()))
+                      .withSideInputs(getSideInputs())
+                      .withOutputTags(writtenRecordsTag, TupleTagList.of(BAD_RECORD_TAG)));
+      addErrorCollection(writtenFiles);
+
+      return writtenFiles.get(writtenRecordsTag).setCoder(fileResultCoder);
     }
   }
 
@@ -763,22 +892,18 @@ public PCollection<List<FileResult<DestinationT>>> expand(PCollection<UserT> inp
       //
       // TODO(https://github.com/apache/beam/issues/20928): The implementation doesn't currently
       // work with merging windows.
+      TupleTag<KV<Integer, UserT>> shardTag = new TupleTag<>("shardTag");
+
+      PCollectionTuple shardedElements =
+          input.apply(
+              "KeyedByDestinationHash",
+              ParDo.of(new KeyByDestinationHash(input.getCoder(), destinationCoder))
+                  .withOutputTags(shardTag, TupleTagList.of(BAD_RECORD_TAG)));
+      addErrorCollection(shardedElements);
+
       PCollection<KV<org.apache.beam.sdk.util.ShardedKey<Integer>, Iterable<UserT>>> shardedInput =
-          input
-              .apply(
-                  "KeyedByDestinationHash",
-                  ParDo.of(
-                      new DoFn<UserT, KV<Integer, UserT>>() {
-                        @ProcessElement
-                        public void processElement(@Element UserT element, ProcessContext context)
-                            throws Exception {
-                          getDynamicDestinations().setSideInputAccessorFromProcessContext(context);
-                          DestinationT destination =
-                              getDynamicDestinations().getDestination(context.element());
-                          context.output(
-                              KV.of(hashDestination(destination, destinationCoder), element));
-                        }
-                      }))
+          shardedElements
+              .get(shardTag)
               .setCoder(KvCoder.of(VarIntCoder.of(), input.getCoder()))
               .apply(
                   "ShardAndBatch",
@@ -791,8 +916,9 @@ public void processElement(@Element UserT element, ProcessContext context)
                       org.apache.beam.sdk.util.ShardedKey.Coder.of(VarIntCoder.of()),
                       IterableCoder.of(input.getCoder())));
 
+      TupleTag<FileResult<DestinationT>> writtenRecordsTag = new TupleTag<>("writtenRecords");
       // Write grouped elements to temp files.
-      PCollection<FileResult<DestinationT>> tempFiles =
+      PCollectionTuple writtenFiles =
           shardedInput
               .apply(
                   "AddDummyShard",
@@ -816,7 +942,15 @@ public KV<ShardedKey<Integer>, Iterable<UserT>> apply(
                       ShardedKeyCoder.of(VarIntCoder.of()), IterableCoder.of(input.getCoder())))
               .apply(
                   "WriteShardsIntoTempFiles",
-                  ParDo.of(new WriteShardsIntoTempFilesFn()).withSideInputs(getSideInputs()))
+                  ParDo.of(new WriteShardsIntoTempFilesFn(input.getCoder()))
+                      .withSideInputs(getSideInputs())
+                      .withOutputTags(writtenRecordsTag, TupleTagList.of(BAD_RECORD_TAG)));
+
+      addErrorCollection(writtenFiles);
+
+      PCollection<FileResult<DestinationT>> tempFiles =
+          writtenFiles
+              .get(writtenRecordsTag)
               .setCoder(fileResultCoder)
               .apply(
                   "DropShardNum",
@@ -865,6 +999,32 @@ public void processElement(
     }
   }
 
+  private class KeyByDestinationHash extends DoFn<UserT, KV<Integer, UserT>> {
+
+    private final Coder<UserT> inputCoder;
+
+    private final Coder<DestinationT> destinationCoder;
+
+    public KeyByDestinationHash(Coder<UserT> inputCoder, Coder<DestinationT> destinationCoder) {
+      this.inputCoder = inputCoder;
+      this.destinationCoder = destinationCoder;
+    }
+
+    @ProcessElement
+    public void processElement(
+        @Element UserT element, ProcessContext context, MultiOutputReceiver outputReceiver)
+        throws Exception {
+      getDynamicDestinations().setSideInputAccessorFromProcessContext(context);
+      MaybeDestination<DestinationT> maybeDestination =
+          getDestinationWithErrorHandling(context.element(), outputReceiver, inputCoder);
+      if (!maybeDestination.isValid) {
+        return;
+      }
+      DestinationT destination = maybeDestination.destination;
+      context.output(KV.of(hashDestination(destination, destinationCoder), element));
+    }
+  }
+
   private class RandomShardingFunction implements ShardingFunction<UserT, DestinationT> {
     private final Coder<DestinationT> destinationCoder;
 
@@ -903,15 +1063,20 @@ private class ApplyShardingFunctionFn extends DoFn<UserT, KV<ShardedKey<Integer>
     private final ShardingFunction<UserT, DestinationT> shardingFn;
     private final @Nullable PCollectionView<Integer> numShardsView;
 
+    private final Coder<UserT> inputCoder;
+
     ApplyShardingFunctionFn(
         ShardingFunction<UserT, DestinationT> shardingFn,
-        @Nullable PCollectionView<Integer> numShardsView) {
+        @Nullable PCollectionView<Integer> numShardsView,
+        Coder<UserT> inputCoder) {
       this.numShardsView = numShardsView;
       this.shardingFn = shardingFn;
+      this.inputCoder = inputCoder;
     }
 
     @ProcessElement
-    public void processElement(ProcessContext context) throws Exception {
+    public void processElement(ProcessContext context, MultiOutputReceiver outputReceiver)
+        throws Exception {
       getDynamicDestinations().setSideInputAccessorFromProcessContext(context);
       final int shardCount;
       if (numShardsView != null) {
@@ -927,7 +1092,12 @@ public void processElement(ProcessContext context) throws Exception {
               + " Got %s",
           shardCount);
 
-      DestinationT destination = getDynamicDestinations().getDestination(context.element());
+      MaybeDestination<DestinationT> maybeDestination =
+          getDestinationWithErrorHandling(context.element(), outputReceiver, inputCoder);
+      if (!maybeDestination.isValid) {
+        return;
+      }
+      DestinationT destination = maybeDestination.destination;
       ShardedKey<Integer> shardKey =
           shardingFn.assignShardKey(destination, context.element(), shardCount);
       context.output(KV.of(shardKey, context.element()));
@@ -936,6 +1106,13 @@ public void processElement(ProcessContext context) throws Exception {
 
   private class WriteShardsIntoTempFilesFn
       extends DoFn<KV<ShardedKey<Integer>, Iterable<UserT>>, FileResult<DestinationT>> {
+
+    private final Coder<UserT> inputCoder;
+
+    public WriteShardsIntoTempFilesFn(Coder<UserT> inputCoder) {
+      this.inputCoder = inputCoder;
+    }
+
     private transient List<CompletionStage<Void>> closeFutures = new ArrayList<>();
     private transient List<KV<Instant, FileResult<DestinationT>>> deferredOutput =
         new ArrayList<>();
@@ -949,14 +1126,21 @@ private void readObject(java.io.ObjectInputStream in)
     }
 
     @ProcessElement
-    public void processElement(ProcessContext c, BoundedWindow window) throws Exception {
+    public void processElement(
+        ProcessContext c, BoundedWindow window, MultiOutputReceiver outputReceiver)
+        throws Exception {
       getDynamicDestinations().setSideInputAccessorFromProcessContext(c);
       // Since we key by a 32-bit hash of the destination, there might be multiple destinations
       // in this iterable. The number of destinations is generally very small (1000s or less), so
       // there will rarely be hash collisions.
       Map<DestinationT, Writer<DestinationT, OutputT>> writers = Maps.newHashMap();
       for (UserT input : c.element().getValue()) {
-        DestinationT destination = getDynamicDestinations().getDestination(input);
+        MaybeDestination<DestinationT> maybeDestination =
+            getDestinationWithErrorHandling(input, outputReceiver, inputCoder);
+        if (!maybeDestination.isValid) {
+          continue;
+        }
+        DestinationT destination = maybeDestination.destination;
         Writer<DestinationT, OutputT> writer = writers.get(destination);
         if (writer == null) {
           String uuid = UUID.randomUUID().toString();
@@ -971,7 +1155,12 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
           writer.open(uuid);
           writers.put(destination, writer);
         }
-        writeOrClose(writer, getDynamicDestinations().formatRecord(input));
+
+        OutputT formattedRecord = formatRecordWithErrorHandling(input, outputReceiver, inputCoder);
+        if (formattedRecord == null) {
+          continue;
+        }
+        writeOrClose(writer, formattedRecord);
       }
 
       // Ensure that we clean-up any prior writers that were being closed as part of this bundle
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/StreamingOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/StreamingOptions.java
index 3fc3c7d4c471..e389ab89cf9c 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/StreamingOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/StreamingOptions.java
@@ -17,6 +17,8 @@
  */
 package org.apache.beam.sdk.options;
 
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 /** Options used to configure streaming. */
 public interface StreamingOptions extends ApplicationNameOptions, PipelineOptions {
   /**
@@ -30,4 +32,13 @@ public interface StreamingOptions extends ApplicationNameOptions, PipelineOption
   boolean isStreaming();
 
   void setStreaming(boolean value);
+
+  @Description(
+      "If set, attempts to produce a pipeline compatible with this prior version of the Beam SDK."
+          + " This string should be interpreted and compared per https://semver.org/."
+          + " See, for example, https://cloud.google.com/dataflow/docs/guides/updating-a-pipeline.")
+  @Nullable
+  String getUpdateCompatibilityVersion();
+
+  void setUpdateCompatibilityVersion(@Nullable String updateCompatibilityVersion);
 }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProvider.java
new file mode 100644
index 000000000000..4b693f883fb7
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProvider.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.providers;
+
+import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
+import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
+
+import com.google.auto.service.AutoService;
+import com.google.auto.value.AutoValue;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.GenerateSequence;
+import org.apache.beam.sdk.providers.GenerateSequenceSchemaTransformProvider.GenerateSequenceConfiguration;
+import org.apache.beam.sdk.schemas.AutoValueSchema;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
+import org.apache.beam.sdk.schemas.annotations.SchemaFieldDescription;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider;
+import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptors;
+import org.joda.time.Duration;
+
+@AutoService(SchemaTransformProvider.class)
+public class GenerateSequenceSchemaTransformProvider
+    extends TypedSchemaTransformProvider<GenerateSequenceConfiguration> {
+  public static final String OUTPUT_ROWS_TAG = "output";
+  public static final Schema OUTPUT_SCHEMA = Schema.builder().addInt64Field("value").build();
+
+  @Override
+  public String identifier() {
+    return "beam:schematransform:org.apache.beam:generate_sequence:v1";
+  }
+
+  @Override
+  public List<String> inputCollectionNames() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public List<String> outputCollectionNames() {
+    return Collections.singletonList(OUTPUT_ROWS_TAG);
+  }
+
+  @Override
+  public String description() {
+    return String.format(
+        "Outputs a PCollection of Beam Rows, each containing a single INT64 "
+            + "number called \"value\". The count is produced from the given \"start\""
+            + "value and either up to the given \"end\" or until 2^63 - 1.%n"
+            + "To produce an unbounded PCollection, simply do not specify an \"end\" value. "
+            + "Unbounded sequences can specify a \"rate\" for output elements.%n"
+            + "In all cases, the sequence of numbers is generated in parallel, so there is no "
+            + "inherent ordering between the generated values");
+  }
+
+  @Override
+  public Class<GenerateSequenceConfiguration> configurationClass() {
+    return GenerateSequenceConfiguration.class;
+  }
+
+  @Override
+  public SchemaTransform from(GenerateSequenceConfiguration configuration) {
+    return new GenerateSequenceSchemaTransform(configuration);
+  }
+
+  @DefaultSchema(AutoValueSchema.class)
+  @AutoValue
+  public abstract static class GenerateSequenceConfiguration {
+    @AutoValue
+    public abstract static class Rate {
+      @SchemaFieldDescription("Number of elements component of the rate.")
+      public abstract Long getElements();
+
+      @SchemaFieldDescription("Number of seconds component of the rate.")
+      @Nullable
+      public abstract Long getSeconds();
+
+      public static Builder builder() {
+        return new AutoValue_GenerateSequenceSchemaTransformProvider_GenerateSequenceConfiguration_Rate
+            .Builder();
+      }
+
+      @AutoValue.Builder
+      public abstract static class Builder {
+        public abstract Builder setElements(Long elements);
+
+        public abstract Builder setSeconds(Long seconds);
+
+        public abstract Rate build();
+      }
+    }
+
+    public static Builder builder() {
+      return new AutoValue_GenerateSequenceSchemaTransformProvider_GenerateSequenceConfiguration
+          .Builder();
+    }
+
+    @SchemaFieldDescription("The minimum number to generate (inclusive).")
+    public abstract Long getStart();
+
+    @SchemaFieldDescription(
+        "The maximum number to generate (exclusive). Will be an unbounded sequence if left unspecified.")
+    @Nullable
+    public abstract Long getEnd();
+
+    @SchemaFieldDescription(
+        "Specifies the rate to generate a given number of elements per a given number of seconds. "
+            + "Applicable only to unbounded sequences.")
+    @Nullable
+    public abstract Rate getRate();
+
+    @AutoValue.Builder
+    public abstract static class Builder {
+
+      public abstract Builder setStart(Long start);
+
+      public abstract Builder setEnd(Long end);
+
+      public abstract Builder setRate(Rate rate);
+
+      public abstract GenerateSequenceConfiguration build();
+    }
+
+    public void validate() {
+      checkNotNull(this.getStart(), "Must specify a starting point \"start\".");
+      Long start = this.getStart();
+      Long end = this.getEnd();
+      if (end != null) {
+        checkArgument(end == -1 || end >= start, "Invalid range [%s, %s)", start, end);
+      }
+      Rate rate = this.getRate();
+      if (rate != null) {
+        checkArgument(
+            rate.getElements() > 0,
+            "Invalid rate specification. Expected positive elements component but received %s.",
+            rate.getElements());
+        checkArgument(
+            Optional.ofNullable(rate.getSeconds()).orElse(1L) > 0,
+            "Invalid rate specification. Expected positive seconds component but received %s.",
+            rate.getSeconds());
+      }
+    }
+  }
+
+  protected static class GenerateSequenceSchemaTransform extends SchemaTransform {
+    private final GenerateSequenceConfiguration configuration;
+
+    GenerateSequenceSchemaTransform(GenerateSequenceConfiguration configuration) {
+      configuration.validate();
+      this.configuration = configuration;
+    }
+
+    @Override
+    public PCollectionRowTuple expand(PCollectionRowTuple input) {
+      checkArgument(
+          input.getAll().isEmpty(), "Expected no inputs but got: %s", input.getAll().keySet());
+
+      Long end = Optional.ofNullable(configuration.getEnd()).orElse(-1L);
+      GenerateSequenceConfiguration.Rate rate = configuration.getRate();
+
+      GenerateSequence sequence = GenerateSequence.from(configuration.getStart()).to(end);
+      if (rate != null) {
+        sequence =
+            sequence.withRate(
+                rate.getElements(),
+                Duration.standardSeconds(Optional.ofNullable(rate.getSeconds()).orElse(1L)));
+      }
+
+      return PCollectionRowTuple.of(
+          OUTPUT_ROWS_TAG,
+          input
+              .getPipeline()
+              .apply(sequence)
+              .apply(
+                  MapElements.into(TypeDescriptors.rows())
+                      .via(l -> Row.withSchema(OUTPUT_SCHEMA).withFieldValue("value", l).build()))
+              .setRowSchema(OUTPUT_SCHEMA));
+    }
+  }
+}
diff --git a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy b/sdks/java/core/src/main/java/org/apache/beam/sdk/providers/package-info.java
similarity index 52%
rename from .test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/providers/package-info.java
index 437d6ab9613c..6d90b7d018ad 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_IT.groovy
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/providers/package-info.java
@@ -16,23 +16,8 @@
  * limitations under the License.
  */
 
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the integration test of python mongodbio class.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_MongoDBIO_IT',
-    'Run Python MongoDBIO_IT', 'Python MongoDBIO Integration Test',this) {
-      description('Runs the Python MongoDBIO Integration Test.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:python:test-suites:direct:mongodbioIT')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
+/**
+ * Defines {@link org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider}s for transforms in
+ * the core module.
+ */
+package org.apache.beam.sdk.providers;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
index e542007c9a55..c76d7a25e69b 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/transforms/SchemaTransformProvider.java
@@ -36,6 +36,15 @@ public interface SchemaTransformProvider {
   /** Returns an id that uniquely represents this transform. */
   String identifier();
 
+  /**
+   * Returns a description regarding the {@link SchemaTransform} represented by the {@link
+   * SchemaTransformProvider}. Please keep the language generic (i.e. not specific to any
+   * programming language). The description may be markdown formatted.
+   */
+  default String description() {
+    return "";
+  }
+
   /**
    * Returns the expected schema of the configuration object. Note this is distinct from the schema
    * of the transform itself.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
index ffbfac460dcd..f1a964fa5a61 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Combine.java
@@ -1555,7 +1555,7 @@ public PerKey<K, InputT, OutputT> withSideInputs(
      */
     public PerKeyWithHotKeyFanout<K, InputT, OutputT> withHotKeyFanout(
         SerializableFunction<? super K, Integer> hotKeyFanout) {
-      return new PerKeyWithHotKeyFanout<>(fn, fnDisplayData, hotKeyFanout, fewKeys);
+      return new PerKeyWithHotKeyFanout<>(fn, fnDisplayData, hotKeyFanout, fewKeys, sideInputs);
     }
 
     /**
@@ -1578,7 +1578,8 @@ public Integer apply(K unused) {
               return hotKeyFanout;
             }
           },
-          fewKeys);
+          fewKeys,
+          sideInputs);
     }
 
     /** Returns the {@link GlobalCombineFn} used by this Combine operation. */
@@ -1624,18 +1625,20 @@ public static class PerKeyWithHotKeyFanout<K, InputT, OutputT>
     private final GlobalCombineFn<? super InputT, ?, OutputT> fn;
     private final DisplayData.ItemSpec<? extends Class<?>> fnDisplayData;
     private final SerializableFunction<? super K, Integer> hotKeyFanout;
-
     private final boolean fewKeys;
+    private final List<PCollectionView<?>> sideInputs;
 
     private PerKeyWithHotKeyFanout(
         GlobalCombineFn<? super InputT, ?, OutputT> fn,
         DisplayData.ItemSpec<? extends Class<?>> fnDisplayData,
         SerializableFunction<? super K, Integer> hotKeyFanout,
-        boolean fewKeys) {
+        boolean fewKeys,
+        List<PCollectionView<?>> sideInputs) {
       this.fn = fn;
       this.fnDisplayData = fnDisplayData;
       this.hotKeyFanout = hotKeyFanout;
       this.fewKeys = fewKeys;
+      this.sideInputs = sideInputs;
     }
 
     @Override
@@ -1928,6 +1931,10 @@ public void processElement(
           fewKeys
               ? Combine.fewKeys(hotPreCombine, fnDisplayData)
               : Combine.perKey(hotPreCombine, fnDisplayData);
+      if (!sideInputs.isEmpty()) {
+        hotPreCombineTransform = hotPreCombineTransform.withSideInputs(sideInputs);
+      }
+
       PCollection<KV<K, InputOrAccum<InputT, AccumT>>> precombinedHot =
           split
               .get(hot)
@@ -1975,6 +1982,10 @@ public KV<K, InputOrAccum<InputT, AccumT>> apply(KV<K, InputT> element) {
           fewKeys
               ? Combine.fewKeys(postCombine, fnDisplayData)
               : Combine.perKey(postCombine, fnDisplayData);
+      if (!sideInputs.isEmpty()) {
+        postCombineTransform = postCombineTransform.withSideInputs(sideInputs);
+      }
+
       return PCollectionList.of(precombinedHot)
           .and(preprocessedCold)
           .apply(Flatten.pCollections())
@@ -1993,6 +2004,11 @@ public void populateDisplayData(DisplayData.Builder builder) {
           DisplayData.item("fanoutFn", hotKeyFanout.getClass()).withLabel("Fanout Function"));
     }
 
+    /** Returns the side inputs used by this Combine operation. */
+    public List<PCollectionView<?>> getSideInputs() {
+      return sideInputs;
+    }
+
     /**
      * Used to store either an input or accumulator value, for flattening the hot and cold key
      * paths.
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
index 079379953cd9..dc81e23b10ee 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Create.java
@@ -55,7 +55,10 @@
 import org.apache.beam.sdk.schemas.Schema;
 import org.apache.beam.sdk.schemas.SchemaCoder;
 import org.apache.beam.sdk.schemas.SchemaRegistry;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -241,6 +244,37 @@ public static <T> TimestampedValues<T> timestamped(
     return timestamped(ImmutableList.<TimestampedValue<T>>builder().add(elem).add(elems).build());
   }
 
+  /**
+   * Returns a new {@link Create.WindowedValues} transform that produces a {@link PCollection}
+   * containing the elements of the provided {@code Iterable} with the specified windowing metadata.
+   *
+   * <p>The argument should not be modified after this is called.
+   *
+   * <p>By default, {@code Create.WindowedValues} can automatically determine the {@code Coder} to
+   * use if all elements have the same non-parameterized run-time class, and a default coder is
+   * registered for that class. See {@link CoderRegistry} for details on how defaults are
+   * determined. Otherwise, use {@link Create.WindowedValues#withCoder} to set the coder explicitly.
+   *
+   * <p>Likewise, the window coder can be inferred if the window type is registered with the {@link
+   * CoderRegistry}. Otherwise, use {@link Create.WindowedValues#withWindowCoder(Coder)} to set the
+   * window coder explicitly.
+   */
+  public static <T> WindowedValues<T> windowedValues(Iterable<WindowedValue<T>> elems) {
+    return new WindowedValues<>(elems, Optional.absent(), Optional.absent(), Optional.absent());
+  }
+
+  /**
+   * Returns a new {@link Create.WindowedValues} transform that produces a {@link PCollection}
+   * containing the specified elements with the specified windowing metadata.
+   *
+   * <p>The arguments should not be modified after this is called.
+   */
+  @SafeVarargs
+  public static <T> WindowedValues<T> windowedValues(
+      WindowedValue<T> elem, @SuppressWarnings("unchecked") WindowedValue<T>... elems) {
+    return windowedValues(ImmutableList.<WindowedValue<T>>builder().add(elem).add(elems).build());
+  }
+
   /**
    * Returns a new root transform that produces a {@link PCollection} containing the specified
    * elements with the specified timestamps.
@@ -727,6 +761,163 @@ public void processElement(@Element TimestampedValue<T> element, OutputReceiver<
     }
   }
 
+  /**
+   * A {@code PTransform} that creates a {@code PCollection} whose elements have associated
+   * windowing metadata.
+   */
+  public static class WindowedValues<T> extends PTransform<PBegin, PCollection<T>> {
+
+    /**
+     * Returns a {@link Create.WindowedValues} PTransform like this one that uses the given {@code
+     * Coder<T>} to decode each of the objects into a value of type {@code T}.
+     *
+     * <p>By default, {@code Create.TimestampedValues} can automatically determine the {@code Coder}
+     * to use if all elements have the same non-parameterized run-time class, and a default coder is
+     * registered for that class. See {@link CoderRegistry} for details on how defaults are
+     * determined.
+     *
+     * <p>Note that for {@link Create.WindowedValues with no elements}, the {@link VoidCoder} is
+     * used.
+     */
+    public WindowedValues<T> withCoder(Coder<T> coder) {
+      return new WindowedValues<>(windowedValues, Optional.of(coder), windowCoder, typeDescriptor);
+    }
+
+    /**
+     * Returns a {@link Create.WindowedValues} PTransform like this one that uses the given {@code
+     * Coder<T>} to decode each of the objects into a value of type {@code T}.
+     *
+     * <p>By default, {@code Create.WindowedValues} can automatically determine the {@code Coder} to
+     * use if all elements have the same non-parameterized run-time class, and a default coder is
+     * registered for that class. See {@link CoderRegistry} for details on how defaults are
+     * determined.
+     *
+     * <p>Note that for {@link Create.WindowedValues with no elements}, the {@link
+     * GlobalWindow.Coder} is used.
+     */
+    public WindowedValues<T> withWindowCoder(Coder<? extends BoundedWindow> windowCoder) {
+      return new WindowedValues<>(
+          windowedValues, elementCoder, Optional.of(windowCoder), typeDescriptor);
+    }
+
+    /**
+     * Returns a {@link Create.WindowedValues} PTransform like this one that uses the given {@code
+     * Schema} to represent objects.
+     */
+    public WindowedValues<T> withSchema(
+        Schema schema,
+        TypeDescriptor<T> typeDescriptor,
+        SerializableFunction<T, Row> toRowFunction,
+        SerializableFunction<Row, T> fromRowFunction) {
+      return withCoder(SchemaCoder.of(schema, typeDescriptor, toRowFunction, fromRowFunction));
+    }
+
+    /**
+     * Returns a {@link Create.WindowedValues} PTransform like this one that uses the given {@code
+     * TypeDescriptor<T>} to determine the {@code Coder} to use to decode each of the objects into a
+     * value of type {@code T}. Note that a default coder must be registered for the class described
+     * in the {@code TypeDescriptor<T>}.
+     *
+     * <p>By default, {@code Create.TimestampedValues} can automatically determine the {@code Coder}
+     * to use if all elements have the same non-parameterized run-time class, and a default coder is
+     * registered for that class. See {@link CoderRegistry} for details on how defaults are
+     * determined.
+     *
+     * <p>Note that for {@link Create.WindowedValues} with no elements, the {@link VoidCoder} is
+     * used.
+     */
+    public WindowedValues<T> withType(TypeDescriptor<T> type) {
+      return new WindowedValues<>(windowedValues, elementCoder, windowCoder, Optional.of(type));
+    }
+
+    @Override
+    public PCollection<T> expand(PBegin input) {
+      try {
+        Coder<T> coder = null;
+        CoderRegistry coderRegistry = input.getPipeline().getCoderRegistry();
+        SchemaRegistry schemaRegistry = input.getPipeline().getSchemaRegistry();
+        if (elementCoder.isPresent()) {
+          coder = elementCoder.get();
+        } else if (typeDescriptor.isPresent()) {
+          try {
+            coder =
+                SchemaCoder.of(
+                    schemaRegistry.getSchema(typeDescriptor.get()),
+                    typeDescriptor.get(),
+                    schemaRegistry.getToRowFunction(typeDescriptor.get()),
+                    schemaRegistry.getFromRowFunction(typeDescriptor.get()));
+          } catch (NoSuchSchemaException e) {
+            // No schema registered.
+          }
+          if (coder == null) {
+            coder = coderRegistry.getCoder(typeDescriptor.get());
+          }
+        } else {
+          Iterable<T> rawElements = Iterables.transform(windowedValues, WindowedValue::getValue);
+          coder = getDefaultCreateCoder(coderRegistry, schemaRegistry, rawElements);
+        }
+
+        Coder<? extends BoundedWindow> windowCoder;
+        if (this.windowCoder.isPresent()) {
+          windowCoder = this.windowCoder.get();
+        } else if (Iterables.isEmpty(windowedValues)) {
+          windowCoder = GlobalWindow.Coder.INSTANCE;
+        } else {
+          Iterable<? extends BoundedWindow> rawWindows =
+              Iterables.concat(Iterables.transform(windowedValues, WindowedValue::getWindows));
+          windowCoder = getDefaultCreateCoder(coderRegistry, schemaRegistry, rawWindows);
+        }
+
+        PCollection<WindowedValue<T>> intermediate =
+            Pipeline.applyTransform(
+                input,
+                Create.of(windowedValues)
+                    .withCoder(WindowedValue.getFullCoder(coder, windowCoder)));
+
+        PCollection<T> output = intermediate.apply(ParDo.of(new ConvertWindowedValues<>()));
+        output.setCoder(coder);
+        return output;
+      } catch (CannotProvideCoderException e) {
+        throw new IllegalArgumentException(
+            "Unable to infer a coder and no Coder was specified. "
+                + "Please set a coder by invoking CreateTimestamped.withCoder() explicitly.",
+            e);
+      }
+    }
+
+    /////////////////////////////////////////////////////////////////////////////
+
+    /** The timestamped elements of the resulting PCollection. */
+    private final transient Iterable<WindowedValue<T>> windowedValues;
+
+    /** The coder used to encode the values to and from a binary representation. */
+    private final transient Optional<Coder<T>> elementCoder;
+
+    private final Optional<Coder<? extends BoundedWindow>> windowCoder;
+
+    /** The value type. */
+    private final transient Optional<TypeDescriptor<T>> typeDescriptor;
+
+    private WindowedValues(
+        Iterable<WindowedValue<T>> windowedValues,
+        Optional<Coder<T>> elementCoder,
+        Optional<Coder<? extends BoundedWindow>> windowCoder,
+        Optional<TypeDescriptor<T>> typeDescriptor) {
+      this.windowedValues = windowedValues;
+      this.elementCoder = elementCoder;
+      this.windowCoder = windowCoder;
+      this.typeDescriptor = typeDescriptor;
+    }
+
+    private static class ConvertWindowedValues<T> extends DoFn<WindowedValue<T>, T> {
+      @ProcessElement
+      public void processElement(@Element WindowedValue<T> element, OutputReceiver<T> r) {
+        r.outputWindowedValue(
+            element.getValue(), element.getTimestamp(), element.getWindows(), element.getPane());
+      }
+    }
+  }
+
   private static <T> Coder<T> getDefaultCreateCoder(
       CoderRegistry coderRegistry, SchemaRegistry schemaRegistry, Iterable<T> elems)
       throws CannotProvideCoderException {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
index 2993d8cca97a..c22b726c99a2 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFn.java
@@ -24,6 +24,7 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import java.util.Collection;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.state.State;
@@ -185,6 +186,31 @@ public abstract class WindowedContext {
      */
     public abstract void outputWithTimestamp(OutputT output, Instant timestamp);
 
+    /**
+     * Adds the given element to the main output {@code PCollection}, with the given windowing
+     * metadata.
+     *
+     * <p>Once passed to {@code outputWindowedValue} the element should not be modified in any way.
+     *
+     * <p>If invoked from {@link ProcessElement}), the timestamp must not be older than the input
+     * element's timestamp minus {@link DoFn#getAllowedTimestampSkew}. The output element will be in
+     * the same windows as the input element.
+     *
+     * <p>If invoked from {@link StartBundle} or {@link FinishBundle}, this will attempt to use the
+     * {@link org.apache.beam.sdk.transforms.windowing.WindowFn} of the input {@code PCollection} to
+     * determine what windows the element should be in, throwing an exception if the {@code
+     * WindowFn} attempts to access any information about the input element except for the
+     * timestamp.
+     *
+     * <p><i>Note:</i> A splittable {@link DoFn} is not allowed to output from {@link StartBundle}
+     * or {@link FinishBundle} methods.
+     */
+    public abstract void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo);
+
     /**
      * Adds the given element to the output {@code PCollection} with the given tag.
      *
@@ -231,6 +257,32 @@ public abstract class WindowedContext {
      * @see ParDo.SingleOutput#withOutputTags
      */
     public abstract <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp);
+
+    /**
+     * Adds the given element to the main output {@code PCollection}, with the given windowing
+     * metadata.
+     *
+     * <p>Once passed to {@code outputWindowedValue} the element should not be modified in any way.
+     *
+     * <p>If invoked from {@link ProcessElement}), the timestamp must not be older than the input
+     * element's timestamp minus {@link DoFn#getAllowedTimestampSkew}. The output element will be in
+     * the same windows as the input element.
+     *
+     * <p>If invoked from {@link StartBundle} or {@link FinishBundle}, this will attempt to use the
+     * {@link org.apache.beam.sdk.transforms.windowing.WindowFn} of the input {@code PCollection} to
+     * determine what windows the element should be in, throwing an exception if the {@code
+     * WindowFn} attempts to access any information about the input element except for the
+     * timestamp.
+     *
+     * <p><i>Note:</i> A splittable {@link DoFn} is not allowed to output from {@link StartBundle}
+     * or {@link FinishBundle} methods.
+     */
+    public abstract <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo);
   }
 
   /** Information accessible when running a {@link DoFn.ProcessElement} method. */
@@ -342,6 +394,12 @@ public interface OutputReceiver<T> {
     void output(T output);
 
     void outputWithTimestamp(T output, Instant timestamp);
+
+    void outputWindowedValue(
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo);
   }
 
   /** Receives tagged output for a multi-output function. */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java
index 4f3719f2cc0e..7adfd7768d34 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnOutputReceivers.java
@@ -20,12 +20,15 @@
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkState;
 
+import java.util.Collection;
 import java.util.Map;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.schemas.SchemaCoder;
 import org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver;
 import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TupleTag;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -58,6 +61,16 @@ public void output(Row output) {
     public void outputWithTimestamp(Row output, Instant timestamp) {
       outputReceiver.outputWithTimestamp(schemaCoder.getFromRowFunction().apply(output), timestamp);
     }
+
+    @Override
+    public void outputWindowedValue(
+        Row output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      outputReceiver.outputWindowedValue(
+          schemaCoder.getFromRowFunction().apply(output), timestamp, windows, paneInfo);
+    }
   }
 
   private static class WindowedContextOutputReceiver<T> implements OutputReceiver<T> {
@@ -87,6 +100,20 @@ public void outputWithTimestamp(T output, Instant timestamp) {
         ((DoFn<?, T>.WindowedContext) context).outputWithTimestamp(output, timestamp);
       }
     }
+
+    @Override
+    public void outputWindowedValue(
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      if (outputTag != null) {
+        context.outputWindowedValue(outputTag, output, timestamp, windows, paneInfo);
+      } else {
+        ((DoFn<?, T>.WindowedContext) context)
+            .outputWindowedValue(output, timestamp, windows, paneInfo);
+      }
+    }
   }
 
   private static class WindowedContextMultiOutputReceiver implements MultiOutputReceiver {
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
index f72de2af35ef..c0915c2dcd75 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/DoFnTester.java
@@ -22,6 +22,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -581,6 +582,15 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
       outputWithTimestamp(mainOutputTag, output, timestamp);
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      outputWindowedValue(mainOutputTag, output, timestamp, windows, paneInfo);
+    }
+
     @Override
     public <T> void output(TupleTag<T> tag, T output) {
       outputWithTimestamp(tag, output, element.getTimestamp());
@@ -591,6 +601,18 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
       getMutableOutput(tag)
           .add(ValueInSingleWindow.of(output, timestamp, element.getWindow(), element.getPane()));
     }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      for (BoundedWindow w : windows) {
+        getMutableOutput(tag).add(ValueInSingleWindow.of(output, timestamp, w, paneInfo));
+      }
+    }
   }
 
   /** @deprecated Use {@link TestPipeline} with the {@code DirectRunner}. */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
index 7b3dfb188033..c0c3638b28dd 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/PTransform.java
@@ -21,6 +21,7 @@
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.CannotProvideCoderException;
@@ -216,6 +217,17 @@ public ResourceHints getResourceHints() {
     return resourceHints;
   }
 
+  /** Returns annotations map to provide additional hints to the runner. */
+  public Map<String, byte[]> getAnnotations() {
+    return annotations;
+  }
+
+  public PTransform<InputT, OutputT> addAnnotation(
+      @NonNull String annotationType, byte @NonNull [] annotation) {
+    annotations.put(annotationType, annotation);
+    return this;
+  }
+
   /////////////////////////////////////////////////////////////////////////////
 
   // See the note about about PTransform's fake Serializability, to
@@ -229,6 +241,8 @@ public ResourceHints getResourceHints() {
 
   protected transient @NonNull ResourceHints resourceHints = ResourceHints.create();
 
+  protected transient @NonNull Map<String, byte @NonNull []> annotations = new HashMap<>();
+
   protected PTransform() {
     this.name = null;
   }
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
index 42f0f6accc7f..a879667ff96f 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Reshuffle.java
@@ -17,8 +17,13 @@
  */
 package org.apache.beam.sdk.transforms;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.ReshuffleTrigger;
 import org.apache.beam.sdk.transforms.windowing.TimestampCombiner;
@@ -27,7 +32,9 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
 import org.apache.beam.sdk.values.WindowingStrategy;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Comparators;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.primitives.UnsignedInteger;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.joda.time.Duration;
@@ -67,6 +74,63 @@ public static <T> ViaRandomKey<T> viaRandomKey() {
 
   @Override
   public PCollection<KV<K, V>> expand(PCollection<KV<K, V>> input) {
+    String requestedVersionString =
+        input.getPipeline().getOptions().as(StreamingOptions.class).getUpdateCompatibilityVersion();
+
+    if (requestedVersionString != null) {
+      List<String> requestedVersion = Arrays.asList(requestedVersionString.split("\\."));
+      List<String> targetVersion = Arrays.asList("2", "53", "0");
+
+      if (Comparators.lexicographical(Comparator.<String>naturalOrder())
+              .compare(requestedVersion, targetVersion)
+          <= 0) {
+        return expand_2_53_0(input);
+      }
+    }
+
+    WindowingStrategy<?, ?> originalStrategy = input.getWindowingStrategy();
+    // If the input has already had its windows merged, then the GBK that performed the merge
+    // will have set originalStrategy.getWindowFn() to InvalidWindows, causing the GBK contained
+    // here to fail. Instead, we install a valid WindowFn that leaves all windows unchanged.
+    // The TimestampCombiner is set to ensure the GroupByKey does not shift elements forwards in
+    // time.
+    // Because this outputs as fast as possible, this should not hold the watermark.
+    Window<KV<K, V>> rewindow =
+        Window.<KV<K, V>>into(new IdentityWindowFn<>(originalStrategy.getWindowFn().windowCoder()))
+            .triggering(new ReshuffleTrigger<>())
+            .discardingFiredPanes()
+            .withTimestampCombiner(TimestampCombiner.EARLIEST)
+            .withAllowedLateness(Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
+
+    PCollection<KV<K, ValueInSingleWindow<V>>> reified =
+        input
+            .apply("SetIdentityWindow", rewindow)
+            .apply("ReifyOriginalMetadata", Reify.windowsInValue());
+
+    PCollection<KV<K, Iterable<ValueInSingleWindow<V>>>> grouped =
+        reified.apply(GroupByKey.create());
+    return grouped
+        .apply(
+            "ExpandIterable",
+            ParDo.of(
+                new DoFn<KV<K, Iterable<ValueInSingleWindow<V>>>, KV<K, ValueInSingleWindow<V>>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<K, Iterable<ValueInSingleWindow<V>>> element,
+                      OutputReceiver<KV<K, ValueInSingleWindow<V>>> r) {
+                    K key = element.getKey();
+                    for (ValueInSingleWindow<V> value : element.getValue()) {
+                      r.output(KV.of(key, value));
+                    }
+                  }
+                }))
+        .apply("RestoreMetadata", new RestoreMetadata<>())
+        // Set the windowing strategy directly, so that it doesn't get counted as the user having
+        // set allowed lateness.
+        .setWindowingStrategyInternal(originalStrategy);
+  }
+
+  private PCollection<KV<K, V>> expand_2_53_0(PCollection<KV<K, V>> input) {
     WindowingStrategy<?, ?> originalStrategy = input.getWindowingStrategy();
     // If the input has already had its windows merged, then the GBK that performed the merge
     // will have set originalStrategy.getWindowFn() to InvalidWindows, causing the GBK contained
@@ -105,6 +169,31 @@ public void processElement(
         .apply("RestoreOriginalTimestamps", ReifyTimestamps.extractFromValues());
   }
 
+  private static class RestoreMetadata<K, V>
+      extends PTransform<PCollection<KV<K, ValueInSingleWindow<V>>>, PCollection<KV<K, V>>> {
+    @Override
+    public PCollection<KV<K, V>> expand(PCollection<KV<K, ValueInSingleWindow<V>>> input) {
+      return input.apply(
+          ParDo.of(
+              new DoFn<KV<K, ValueInSingleWindow<V>>, KV<K, V>>() {
+                @Override
+                public Duration getAllowedTimestampSkew() {
+                  return Duration.millis(Long.MAX_VALUE);
+                }
+
+                @ProcessElement
+                public void processElement(
+                    @Element KV<K, ValueInSingleWindow<V>> kv, OutputReceiver<KV<K, V>> r) {
+                  r.outputWindowedValue(
+                      KV.of(kv.getKey(), kv.getValue().getValue()),
+                      kv.getValue().getTimestamp(),
+                      Collections.singleton(kv.getValue().getWindow()),
+                      kv.getValue().getPane());
+                }
+              }));
+    }
+  }
+
   /** Implementation of {@link #viaRandomKey()}. */
   public static class ViaRandomKey<T> extends PTransform<PCollection<T>, PCollection<T>> {
     private ViaRandomKey() {}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/BadRecord.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/BadRecord.java
new file mode 100644
index 000000000000..fd49078350c4
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/BadRecord.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.auto.value.AutoValue;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.Serializable;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.schemas.AutoValueSchema;
+import org.apache.beam.sdk.schemas.NoSuchSchemaException;
+import org.apache.beam.sdk.schemas.SchemaCoder;
+import org.apache.beam.sdk.schemas.SchemaRegistry;
+import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
+import org.apache.beam.sdk.util.CoderUtils;
+import org.apache.beam.sdk.util.Preconditions;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@AutoValue
+@DefaultSchema(AutoValueSchema.class)
+public abstract class BadRecord implements Serializable {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BadRecord.class);
+
+  /** Information about the record that failed. */
+  public abstract Record getRecord();
+
+  /** Information about why the record failed. */
+  public abstract Failure getFailure();
+
+  public static Builder builder() {
+    return new AutoValue_BadRecord.Builder();
+  }
+
+  public static Coder<BadRecord> getCoder(Pipeline pipeline) {
+    try {
+      SchemaRegistry schemaRegistry = pipeline.getSchemaRegistry();
+      return SchemaCoder.of(
+          schemaRegistry.getSchema(BadRecord.class),
+          TypeDescriptor.of(BadRecord.class),
+          schemaRegistry.getToRowFunction(BadRecord.class),
+          schemaRegistry.getFromRowFunction(BadRecord.class));
+    } catch (NoSuchSchemaException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static <RecordT> BadRecord fromExceptionInformation(
+      RecordT record,
+      @Nullable Coder<RecordT> coder,
+      @Nullable Exception exception,
+      String description)
+      throws IOException {
+    Preconditions.checkArgumentNotNull(record);
+
+    // Build up record information
+    BadRecord.Record.Builder recordBuilder = Record.builder();
+    recordBuilder.addHumanReadableJson(record).addCoderAndEncodedRecord(coder, record);
+
+    // Build up failure information
+    BadRecord.Failure.Builder failureBuilder = Failure.builder().setDescription(description);
+
+    // It's possible for us to want to handle an error scenario where no actual exception object
+    // exists
+    if (exception != null) {
+      failureBuilder.setException(exception.toString()).addExceptionStackTrace(exception);
+    }
+
+    return BadRecord.builder()
+        .setRecord(recordBuilder.build())
+        .setFailure(failureBuilder.build())
+        .build();
+  }
+
+  @AutoValue.Builder
+  public abstract static class Builder {
+
+    public abstract Builder setRecord(Record record);
+
+    public abstract Builder setFailure(Failure error);
+
+    public abstract BadRecord build();
+  }
+
+  @AutoValue
+  @DefaultSchema(AutoValueSchema.class)
+  public abstract static class Record implements Serializable {
+
+    /** The failing record, encoded as JSON. Will be null if serialization as JSON fails. */
+    public abstract @Nullable String getHumanReadableJsonRecord();
+
+    /**
+     * Nullable to account for failing to encode, or if there is no coder for the record at the time
+     * of failure.
+     */
+    @SuppressWarnings("mutable")
+    public abstract byte @Nullable [] getEncodedRecord();
+
+    /** The coder for the record, or null if there is no coder. */
+    public abstract @Nullable String getCoder();
+
+    public static Builder builder() {
+      return new AutoValue_BadRecord_Record.Builder();
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder {
+
+      public abstract Builder setHumanReadableJsonRecord(@Nullable String jsonRecord);
+
+      public Builder addHumanReadableJson(Object record) {
+        ObjectWriter objectWriter = new ObjectMapper().writer().withDefaultPrettyPrinter();
+        try {
+          this.setHumanReadableJsonRecord(objectWriter.writeValueAsString(record));
+        } catch (Exception e) {
+          LOG.error(
+              "Unable to serialize record as JSON. Human readable record attempted via .toString",
+              e);
+          try {
+            this.setHumanReadableJsonRecord(record.toString());
+          } catch (Exception e2) {
+            LOG.error(
+                "Unable to serialize record via .toString. Human readable record will be null", e2);
+          }
+        }
+        return this;
+      }
+
+      @SuppressWarnings("mutable")
+      public abstract Builder setEncodedRecord(byte @Nullable [] encodedRecord);
+
+      public abstract Builder setCoder(@Nullable String coder);
+
+      public <T> Builder addCoderAndEncodedRecord(@Nullable Coder<T> coder, T record) {
+        // We will sometimes not have a coder for a failing record, for example if it has already
+        // been
+        // modified within the dofn.
+        if (coder != null) {
+          this.setCoder(coder.toString());
+          try {
+            this.setEncodedRecord(CoderUtils.encodeToByteArray(coder, record));
+          } catch (IOException e) {
+            LOG.error(
+                "Unable to encode failing record using provided coder."
+                    + " BadRecord will be published without encoded bytes",
+                e);
+          }
+        }
+        return this;
+      }
+
+      public abstract Record build();
+    }
+  }
+
+  @AutoValue
+  @DefaultSchema(AutoValueSchema.class)
+  public abstract static class Failure implements Serializable {
+
+    /** The exception itself, e.g. IOException. Null if there is a failure without an exception. */
+    public abstract @Nullable String getException();
+
+    /** The full stacktrace. Null if there is a failure without an exception. */
+    public abstract @Nullable String getExceptionStacktrace();
+
+    /** The description of what was being attempted when the failure occurred. */
+    public abstract String getDescription();
+
+    public static Builder builder() {
+      return new AutoValue_BadRecord_Failure.Builder();
+    }
+
+    @AutoValue.Builder
+    public abstract static class Builder {
+
+      public abstract Builder setException(@Nullable String exception);
+
+      public abstract Builder setExceptionStacktrace(@Nullable String stacktrace);
+
+      public Builder addExceptionStackTrace(Exception exception) throws IOException {
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(stream, false, Charsets.UTF_8.name());
+        exception.printStackTrace(printStream);
+        printStream.close();
+
+        this.setExceptionStacktrace(new String(stream.toByteArray(), Charsets.UTF_8));
+        return this;
+      }
+
+      public abstract Builder setDescription(String description);
+
+      public abstract Failure build();
+    }
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/BadRecordRouter.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/BadRecordRouter.java
new file mode 100644
index 000000000000..7670baf5c6ff
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/BadRecordRouter.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import java.io.Serializable;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver;
+import org.apache.beam.sdk.util.Preconditions;
+import org.apache.beam.sdk.values.TupleTag;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+public interface BadRecordRouter extends Serializable {
+
+  BadRecordRouter THROWING_ROUTER = new ThrowingBadRecordRouter();
+
+  BadRecordRouter RECORDING_ROUTER = new RecordingBadRecordRouter();
+
+  TupleTag<BadRecord> BAD_RECORD_TAG = new TupleTag<>();
+
+  <RecordT> void route(
+      MultiOutputReceiver outputReceiver,
+      RecordT record,
+      @Nullable Coder<RecordT> coder,
+      @Nullable Exception exception,
+      String description)
+      throws Exception;
+
+  class ThrowingBadRecordRouter implements BadRecordRouter {
+
+    @Override
+    public <RecordT> void route(
+        MultiOutputReceiver outputReceiver,
+        RecordT record,
+        @Nullable Coder<RecordT> coder,
+        @Nullable Exception exception,
+        String description)
+        throws Exception {
+      if (exception != null) {
+        throw exception;
+      } else {
+        Preconditions.checkArgumentNotNull(record);
+        String encodedRecord =
+            BadRecord.Record.builder()
+                .addHumanReadableJson(record)
+                .build()
+                .getHumanReadableJsonRecord();
+        if (encodedRecord == null) {
+          encodedRecord = "Unable to serialize bad record";
+        }
+        throw new RuntimeException("Encountered Bad Record: " + encodedRecord);
+      }
+    }
+  }
+
+  class RecordingBadRecordRouter implements BadRecordRouter {
+
+    @Override
+    public <RecordT> void route(
+        MultiOutputReceiver outputReceiver,
+        RecordT record,
+        @Nullable Coder<RecordT> coder,
+        @Nullable Exception exception,
+        String description)
+        throws Exception {
+      outputReceiver
+          .get(BAD_RECORD_TAG)
+          .output(BadRecord.fromExceptionInformation(record, coder, exception, description));
+    }
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandler.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandler.java
new file mode 100644
index 000000000000..cf040470d608
--- /dev/null
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandler.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.POutput;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An Error Handler is a utility object used for plumbing error PCollections to a configured sink
+ * Error Handlers must be closed before a pipeline is run to properly pipe error collections to the
+ * sink, and the pipeline will be rejected if any handlers aren't closed.
+ *
+ * @param <ErrorT> The type of the error object. This will usually be a {@link BadRecord}, but can
+ *     be any type
+ * @param <OutputT> The return type of the sink PTransform.
+ *     <p>Usage of Error Handlers:
+ *     <p>Simple usage with one DLQ
+ *     <pre>{@code
+ * PCollection<?> records = ...;
+ * try (BadRecordErrorHandler<T> errorHandler = pipeline.registerBadRecordErrorHandler(SomeSink.write())) {
+ *  PCollection<?> results = records.apply(SomeIO.write().withErrorHandler(errorHandler));
+ * }
+ * results.apply(SomeOtherTransform);
+ * }</pre>
+ *     Usage with multiple DLQ stages
+ *     <pre>{@code
+ * PCollection<?> records = ...;
+ * try (BadRecordErrorHandler<T> errorHandler = pipeline.registerBadRecordErrorHandler(SomeSink.write())) {
+ *  PCollection<?> results = records.apply(SomeIO.write().withErrorHandler(errorHandler))
+ *                        .apply(OtherTransform.builder().withErrorHandler(errorHandler));
+ * }
+ * results.apply(SomeOtherTransform);
+ * }</pre>
+ *     This is marked as serializable despite never being needed on the runner, to enable it to be a
+ *     parameter of an Autovalue configured PTransform.
+ */
+public interface ErrorHandler<ErrorT, OutputT extends POutput> extends AutoCloseable, Serializable {
+
+  void addErrorCollection(PCollection<ErrorT> errorCollection);
+
+  boolean isClosed();
+
+  @Nullable
+  OutputT getOutput();
+
+  class PTransformErrorHandler<ErrorT, OutputT extends POutput>
+      implements ErrorHandler<ErrorT, OutputT> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PTransformErrorHandler.class);
+    private final PTransform<PCollection<ErrorT>, OutputT> sinkTransform;
+
+    // transient as Pipelines are not serializable
+    private final transient Pipeline pipeline;
+
+    private final Coder<ErrorT> coder;
+
+    // transient as PCollections are not serializable
+    private transient List<PCollection<ErrorT>> errorCollections = new ArrayList<>();
+
+    // transient as PCollections are not serializable
+    private transient @Nullable OutputT sinkOutput = null;
+
+    private boolean closed = false;
+
+    /**
+     * Constructs a new ErrorHandler, but should not be called directly. Instead, call
+     * pipeline.registerErrorHandler to ensure safe pipeline construction
+     */
+    @Internal
+    public PTransformErrorHandler(
+        PTransform<PCollection<ErrorT>, OutputT> sinkTransform,
+        Pipeline pipeline,
+        Coder<ErrorT> coder) {
+      this.sinkTransform = sinkTransform;
+      this.pipeline = pipeline;
+      this.coder = coder;
+    }
+
+    private void readObject(ObjectInputStream aInputStream)
+        throws ClassNotFoundException, IOException {
+      aInputStream.defaultReadObject();
+      errorCollections = new ArrayList<>();
+    }
+
+    @Override
+    public void addErrorCollection(PCollection<ErrorT> errorCollection) {
+      if (isClosed()) {
+        throw new IllegalStateException(
+            "Error collections cannot be added after Error Handler is closed");
+      }
+      errorCollections.add(errorCollection);
+    }
+
+    @Override
+    public boolean isClosed() {
+      return closed;
+    }
+
+    @Override
+    public @Nullable OutputT getOutput() {
+      if (!this.isClosed()) {
+        throw new IllegalStateException(
+            "ErrorHandler must be finalized before the output can be returned");
+      }
+      return sinkOutput;
+    }
+
+    @Override
+    public void close() {
+      if (closed) {
+        throw new IllegalStateException(
+            "Error handler is already closed, and may not be closed twice");
+      }
+      closed = true;
+      PCollection<ErrorT> flattened;
+      if (errorCollections.isEmpty()) {
+        LOG.info("Empty list of error pcollections passed to ErrorHandler.");
+        flattened = pipeline.apply(Create.empty(coder));
+      } else {
+        flattened = PCollectionList.of(errorCollections).apply(Flatten.pCollections());
+      }
+      LOG.debug(
+          "{} error collections are being sent to {}",
+          errorCollections.size(),
+          sinkTransform.getName());
+      String sinkTransformName = sinkTransform.getName();
+      sinkOutput =
+          flattened
+              .apply(
+                  "Record Error Metrics to " + sinkTransformName,
+                  new WriteErrorMetrics<ErrorT>(sinkTransformName))
+              .apply(
+                  "Write to error Sink",
+                  sinkTransform.addAnnotation(
+                      "FeatureMetric", "ErrorHandler".getBytes(StandardCharsets.UTF_8)));
+    }
+
+    public static class WriteErrorMetrics<ErrorT>
+        extends PTransform<PCollection<ErrorT>, PCollection<ErrorT>> {
+
+      private final Counter errorCounter;
+
+      public WriteErrorMetrics(String sinkTransformName) {
+        errorCounter = Metrics.counter("ErrorMetrics", sinkTransformName + "-input");
+      }
+
+      @Override
+      public PCollection<ErrorT> expand(PCollection<ErrorT> input) {
+        return input.apply(ParDo.of(new CountErrors<ErrorT>(errorCounter)));
+      }
+
+      public static class CountErrors<ErrorT> extends DoFn<ErrorT, ErrorT> {
+
+        private final Counter errorCounter;
+
+        public CountErrors(Counter errorCounter) {
+          this.errorCounter = errorCounter;
+        }
+
+        @ProcessElement
+        public void processElement(@Element ErrorT error, OutputReceiver<ErrorT> receiver) {
+          errorCounter.inc();
+          receiver.output(error);
+        }
+      }
+    }
+  }
+
+  class BadRecordErrorHandler<OutputT extends POutput>
+      extends PTransformErrorHandler<BadRecord, OutputT> {
+
+    /** Constructs a new ErrorHandler for handling BadRecords. */
+    @Internal
+    public BadRecordErrorHandler(
+        PTransform<PCollection<BadRecord>, OutputT> sinkTransform, Pipeline pipeline) {
+      super(sinkTransform, pipeline, BadRecord.getCoder(pipeline));
+    }
+  }
+
+  /**
+   * A default, placeholder error handler that exists to allow usage of .addErrorCollection()
+   * without effects. This enables more simple codepaths without checking for whether the user
+   * configured an error handler or not.
+   */
+  @Internal
+  class DefaultErrorHandler<ErrorT, OutputT extends POutput>
+      implements ErrorHandler<ErrorT, OutputT> {
+
+    @Override
+    public void addErrorCollection(PCollection<ErrorT> errorCollection) {}
+
+    @Override
+    public boolean isClosed() {
+      throw new IllegalArgumentException(
+          "No Op handler should not be closed. This implies this IO is misconfigured.");
+    }
+
+    @Override
+    public @Nullable OutputT getOutput() {
+      throw new IllegalArgumentException(
+          "No Op handler has no output. This implies this IO is misconfigured.");
+    }
+
+    @Override
+    public void close() {
+      throw new IllegalArgumentException(
+          "No Op handler should not be closed. This implies this IO is misconfigured.");
+    }
+  }
+}
diff --git a/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Spark.groovy b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/package-info.java
similarity index 52%
rename from .test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Spark.groovy
rename to sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/package-info.java
index 383d4234749c..c41e10143c12 100644
--- a/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Spark.groovy
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/errorhandling/package-info.java
@@ -15,24 +15,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+/** Provides utilities for handling errors in Pipelines. */
+@DefaultAnnotation(NonNull.class)
+package org.apache.beam.sdk.transforms.errorhandling;
 
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
-
-// This job runs the suite of Go integration tests against the Spark runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go_VR_Spark',
-    'Run Go Spark ValidatesRunner', 'Go Spark ValidatesRunner Tests', this) {
-      description('Runs Go integration tests on the Spark runner.')
-
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
-
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:go:test:sparkValidatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
+import edu.umd.cs.findbugs.annotations.DefaultAnnotation;
+import org.checkerframework.checker.nullness.qual.NonNull;
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
index 28b92b390297..cadb2b33bb2a 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/windowing/GlobalWindow.java
@@ -54,6 +54,11 @@ public int hashCode() {
     return GlobalWindow.class.hashCode();
   }
 
+  @Override
+  public String toString() {
+    return "GlobalWindow";
+  }
+
   private GlobalWindow() {}
 
   /** {@link Coder} for encoding and decoding {@code GlobalWindow}s. */
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java
index 2dd44ce27bbc..b235e774feda 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/HistogramData.java
@@ -46,7 +46,9 @@ public class HistogramData implements Serializable {
   private long[] buckets;
   private long numBoundedBucketRecords;
   private long numTopRecords;
+  private double topRecordsSum;
   private long numBottomRecords;
+  private double bottomRecordsSum;
 
   @GuardedBy("this")
   private double sumOfSquaredDeviations;
@@ -64,7 +66,9 @@ public HistogramData(BucketType bucketType) {
     this.buckets = new long[bucketType.getNumBuckets()];
     this.numBoundedBucketRecords = 0;
     this.numTopRecords = 0;
+    this.topRecordsSum = 0;
     this.numBottomRecords = 0;
+    this.bottomRecordsSum = 0;
     this.mean = 0;
     this.sumOfSquaredDeviations = 0;
   }
@@ -151,7 +155,9 @@ public synchronized void update(HistogramData other) {
       }
 
       incTopBucketCount(other.numTopRecords);
+      this.topRecordsSum = other.topRecordsSum;
       incBottomBucketCount(other.numBottomRecords);
+      this.bottomRecordsSum = other.bottomRecordsSum;
       for (int i = 0; i < other.buckets.length; i++) {
         incBucketCount(i, other.buckets[i]);
       }
@@ -181,18 +187,32 @@ public synchronized void clear() {
     this.buckets = new long[bucketType.getNumBuckets()];
     this.numBoundedBucketRecords = 0;
     this.numTopRecords = 0;
+    this.topRecordsSum = 0;
     this.numBottomRecords = 0;
+    this.bottomRecordsSum = 0;
     this.mean = 0;
     this.sumOfSquaredDeviations = 0;
   }
 
+  /**
+   * Copies all updates to a new histogram object and resets 'this' histogram.
+   *
+   * @return New histogram object that has the the same updates as 'this'.
+   */
+  public synchronized HistogramData getAndReset() {
+    HistogramData other = new HistogramData(this.getBucketType());
+    other.update(this);
+    this.clear();
+    return other;
+  }
+
   public synchronized void record(double value) {
     double rangeTo = bucketType.getRangeTo();
     double rangeFrom = bucketType.getRangeFrom();
     if (value >= rangeTo) {
-      numTopRecords++;
+      recordTopRecordsValue(value);
     } else if (value < rangeFrom) {
-      numBottomRecords++;
+      recordBottomRecordsValue(value);
     } else {
       buckets[bucketType.getBucketIndex(value)]++;
       numBoundedBucketRecords++;
@@ -219,6 +239,30 @@ private synchronized void updateStatistics(double value) {
     sumOfSquaredDeviations += (value - mean) * (value - oldMean);
   }
 
+  /**
+   * Increment the {@code numTopRecords} and update {@code topRecordsSum} when a new overflow value
+   * is recorded. This function should only be called when a Histogram is recording a value greater
+   * than the upper bound of it's largest bucket.
+   *
+   * @param value
+   */
+  private synchronized void recordTopRecordsValue(double value) {
+    numTopRecords++;
+    topRecordsSum += value;
+  }
+
+  /**
+   * Increment the {@code numBottomRecords} and update {@code bottomRecordsSum} when a new underflow
+   * value is recorded. This function should only be called when a Histogram is recording a value
+   * smaller than the lowerbound bound of it's smallest bucket.
+   *
+   * @param value
+   */
+  private synchronized void recordBottomRecordsValue(double value) {
+    numBottomRecords++;
+    bottomRecordsSum += value;
+  }
+
   public synchronized long getTotalCount() {
     return numBoundedBucketRecords + numTopRecords + numBottomRecords;
   }
@@ -248,10 +292,18 @@ public synchronized long getTopBucketCount() {
     return numTopRecords;
   }
 
+  public synchronized double getTopBucketMean() {
+    return numTopRecords == 0 ? 0 : topRecordsSum / numTopRecords;
+  }
+
   public synchronized long getBottomBucketCount() {
     return numBottomRecords;
   }
 
+  public synchronized double getBottomBucketMean() {
+    return numBottomRecords == 0 ? 0 : bottomRecordsSum / numBottomRecords;
+  }
+
   public synchronized double getMean() {
     return mean;
   }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
index 64a722495667..d3fcfb291fca 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileSystemsTest.java
@@ -42,6 +42,7 @@
 import org.apache.beam.sdk.io.fs.ResourceId;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.util.MimeTypes;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.FluentIterable;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets;
@@ -316,6 +317,25 @@ public void testInvalidSchemaMatchNewResource() {
     assertEquals("file", FileSystems.matchNewResource("c:/tmp/f1", false));
   }
 
+  @Test
+  public void testMatchNewDirectory() {
+    List<KV<String, KV<String, String[]>>> testCases =
+        ImmutableList.<KV<String, KV<String, String[]>>>builder()
+            .add(KV.of("/abc/d/", KV.of("/abc", new String[] {"d"})))
+            .add(KV.of("/abc/d/", KV.of("/abc/", new String[] {"d"})))
+            .add(KV.of("/abc/d/", KV.of("/abc", new String[] {"d/"})))
+            .add(KV.of("/abc/d/e/f/", KV.of("/abc", new String[] {"d", "e", "f"})))
+            .add(KV.of("/abc/", KV.of("/abc", new String[] {})))
+            .build();
+    for (KV<String, KV<String, String[]>> testCase : testCases) {
+      ResourceId expected = FileSystems.matchNewResource(testCase.getKey(), true);
+      ResourceId actual =
+          FileSystems.matchNewDirectory(
+              testCase.getValue().getKey(), testCase.getValue().getValue());
+      assertEquals(expected, actual);
+    }
+  }
+
   private static List<ResourceId> toResourceIds(List<Path> paths, final boolean isDirectory) {
     return FluentIterable.from(paths)
         .transform(path -> (ResourceId) LocalResourceId.fromPath(path, isDirectory))
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java
index 39cb612f2d89..2db20b92f27f 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/WriteFilesTest.java
@@ -61,6 +61,7 @@
 import org.apache.beam.sdk.options.PipelineOptionsFactoryTest.TestPipelineOptions;
 import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestStream;
 import org.apache.beam.sdk.testing.UsesTestStream;
@@ -78,6 +79,8 @@
 import org.apache.beam.sdk.transforms.Values;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandlingTestUtils.ErrorSinkTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
@@ -634,6 +637,134 @@ private void testDynamicDestinationsHelper(boolean bounded, boolean emptyShards)
     }
   }
 
+  // Test FailingDynamicDestinations class. Expects user values to be string-encoded integers.
+  // Throws exceptions when trying to format records or get destinations based on the mod
+  // of the element
+  static class FailingTestDestinations extends DynamicDestinations<String, Integer, String> {
+    private ResourceId baseOutputDirectory;
+
+    FailingTestDestinations(ResourceId baseOutputDirectory) {
+      this.baseOutputDirectory = baseOutputDirectory;
+    }
+
+    @Override
+    public String formatRecord(String record) {
+      int value = Integer.valueOf(record);
+      // deterministically fail to format 1/3rd of records
+      if (value % 3 == 0) {
+        throw new RuntimeException("Failed To Format Record");
+      }
+      return "record_" + record;
+    }
+
+    @Override
+    public Integer getDestination(String element) {
+      int value = Integer.valueOf(element);
+      // deterministically fail to find the destination for 1/7th of records
+      if (value % 7 == 0) {
+        throw new RuntimeException("Failed To Get Destination");
+      }
+      return value % 5;
+    }
+
+    @Override
+    public Integer getDefaultDestination() {
+      return 0;
+    }
+
+    @Override
+    public FilenamePolicy getFilenamePolicy(Integer destination) {
+      return new PerWindowFiles(
+          baseOutputDirectory.resolve("file_" + destination, StandardResolveOptions.RESOLVE_FILE),
+          "simple");
+    }
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void testFailingDynamicDestinationsBounded() throws Exception {
+    testFailingDynamicDestinationsHelper(true, false);
+  }
+
+  @Test
+  @Category({NeedsRunner.class, UsesUnboundedPCollections.class})
+  public void testFailingDynamicDestinationsUnbounded() throws Exception {
+    testFailingDynamicDestinationsHelper(false, false);
+  }
+
+  @Test
+  @Category({NeedsRunner.class, UsesUnboundedPCollections.class})
+  public void testFailingDynamicDestinationsAutosharding() throws Exception {
+    testFailingDynamicDestinationsHelper(false, true);
+  }
+
+  private void testFailingDynamicDestinationsHelper(boolean bounded, boolean autosharding)
+      throws IOException {
+    FailingTestDestinations dynamicDestinations =
+        new FailingTestDestinations(getBaseOutputDirectory());
+    SimpleSink<Integer> sink =
+        new SimpleSink<>(getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED);
+
+    // Flag to validate that the pipeline options are passed to the Sink.
+    WriteOptions options = TestPipeline.testingPipelineOptions().as(WriteOptions.class);
+    options.setTestFlag("test_value");
+    Pipeline p = TestPipeline.create(options);
+
+    final int numInputs = 100;
+    long expectedFailures = 0;
+    List<String> inputs = Lists.newArrayList();
+    for (int i = 0; i < numInputs; ++i) {
+      inputs.add(Integer.toString(i));
+      if (i % 7 == 0 || i % 3 == 0) {
+        expectedFailures++;
+      }
+    }
+    // Prepare timestamps for the elements.
+    List<Long> timestamps = new ArrayList<>();
+    for (long i = 0; i < inputs.size(); i++) {
+      timestamps.add(i + 1);
+    }
+
+    BadRecordErrorHandler<PCollection<Long>> errorHandler =
+        p.registerBadRecordErrorHandler(new ErrorSinkTransform());
+    int numShards = autosharding ? 0 : 2;
+    WriteFiles<String, Integer, String> writeFiles =
+        WriteFiles.to(sink).withNumShards(numShards).withBadRecordErrorHandler(errorHandler);
+
+    PCollection<String> input = p.apply(Create.timestamped(inputs, timestamps));
+    WriteFilesResult<Integer> res;
+    if (!bounded) {
+      input.setIsBoundedInternal(IsBounded.UNBOUNDED);
+      input = input.apply(Window.into(FixedWindows.of(Duration.standardDays(1))));
+      res = input.apply(writeFiles.withWindowedWrites());
+    } else {
+      res = input.apply(writeFiles);
+    }
+
+    errorHandler.close();
+
+    PAssert.thatSingleton(errorHandler.getOutput()).isEqualTo(expectedFailures);
+
+    res.getPerDestinationOutputFilenames().apply(new VerifyFilesExist<>());
+    p.run();
+
+    for (int i = 0; i < 5; ++i) {
+      ResourceId base =
+          getBaseOutputDirectory().resolve("file_" + i, StandardResolveOptions.RESOLVE_FILE);
+      List<String> expected = Lists.newArrayList();
+      for (int j = i; j < numInputs; j += 5) {
+        if (j % 3 != 0 && j % 7 != 0) {
+          expected.add("record_" + j);
+        }
+      }
+      checkFileContents(
+          base.toString(),
+          expected,
+          Optional.fromNullable(autosharding ? null : numShards),
+          bounded /* expectRemovedTempDirectory */);
+    }
+  }
+
   @Test
   public void testShardedDisplayData() {
     DynamicDestinations<String, Void, String> dynamicDestinations =
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProviderTest.java
new file mode 100644
index 000000000000..dcff3dedb843
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProviderTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.providers;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.providers.GenerateSequenceSchemaTransformProvider.GenerateSequenceConfiguration;
+import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.values.PCollectionRowTuple;
+import org.apache.beam.sdk.values.Row;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class GenerateSequenceSchemaTransformProviderTest {
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void testGenerateSequence() {
+    GenerateSequenceConfiguration config =
+        GenerateSequenceConfiguration.builder().setStart(0L).setEnd(10L).build();
+    SchemaTransform sequence = new GenerateSequenceSchemaTransformProvider().from(config);
+
+    List<Row> expected = new ArrayList<>(10);
+    for (long i = 0L; i < 10L; i++) {
+      expected.add(
+          Row.withSchema(GenerateSequenceSchemaTransformProvider.OUTPUT_SCHEMA)
+              .withFieldValue("value", i)
+              .build());
+    }
+
+    PAssert.that(
+            PCollectionRowTuple.empty(p)
+                .apply(sequence)
+                .get(GenerateSequenceSchemaTransformProvider.OUTPUT_ROWS_TAG))
+        .containsInAnyOrder(expected);
+    p.run();
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
index db7b1436a128..2b698f4f67bb 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/schemas/transforms/TypedSchemaTransformProviderTest.java
@@ -61,6 +61,11 @@ public String identifier() {
       return "fake:v1";
     }
 
+    @Override
+    public String description() {
+      return "Description of fake provider";
+    }
+
     @Override
     protected Class<Configuration> configurationClass() {
       return Configuration.class;
@@ -115,6 +120,7 @@ public void testFrom() {
     Configuration outputConfig = ((FakeSchemaTransform) provider.from(inputConfig)).config;
     assertEquals("field1", outputConfig.getField1());
     assertEquals(13, outputConfig.getField2().intValue());
+    assertEquals("Description of fake provider", provider.description());
   }
 
   @Test
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
index 024fedd177e5..f070378a64ee 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CombineTest.java
@@ -1031,6 +1031,30 @@ public void testWithFanoutPreservesSideInputs() {
 
       assertEquals(Collections.singletonList(view), combine.getSideInputs());
     }
+
+    @Test
+    @Category({ValidatesRunner.class, UsesSideInputs.class})
+    public void testHotKeyCombineWithSideInputs() {
+      PCollection<KV<String, Integer>> input =
+          createInput(
+              pipeline,
+              Arrays.asList(
+                  KV.of("a", 1), KV.of("a", 1), KV.of("a", 4), KV.of("b", 1), KV.of("b", 13)));
+      PCollection<Integer> sum =
+          input.apply(Values.create()).apply("Sum", Combine.globally(new SumInts()));
+      PCollectionView<Integer> sumView = sum.apply(View.asSingleton());
+
+      PCollection<KV<String, String>> combinePerKeyWithSideInputsAndHotKey =
+          input.apply(
+              Combine.<String, Integer, String>perKey(new TestCombineFnWithContext(sumView))
+                  .withSideInputs(sumView)
+                  .withHotKeyFanout(1));
+
+      PAssert.that(combinePerKeyWithSideInputsAndHotKey)
+          .containsInAnyOrder(Arrays.asList(KV.of("a", "20:114"), KV.of("b", "20:113")));
+
+      pipeline.run();
+    }
   }
 
   /** Tests validating windowing behaviors. */
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
index 689e5af7055b..85c8d0d04ede 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/CreateTest.java
@@ -35,6 +35,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
+import java.util.stream.Collectors;
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
 import org.apache.beam.sdk.coders.Coder;
@@ -57,7 +58,11 @@
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.ValidatesRunner;
 import org.apache.beam.sdk.transforms.Create.Values.CreateSource;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.SerializableUtils;
+import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.Row;
@@ -240,6 +245,22 @@ public void processElement(ProcessContext c) {
     }
   }
 
+  private static class FormatMetadata extends DoFn<String, String> {
+    @ProcessElement
+    public void processElement(
+        @Element String e,
+        @Timestamp Instant timestamp,
+        BoundedWindow w,
+        PaneInfo p,
+        OutputReceiver<String> o) {
+      o.output(formatMetadata(e, timestamp, w, p));
+    }
+  }
+
+  private static String formatMetadata(String s, Instant timestamp, BoundedWindow w, PaneInfo p) {
+    return s + ":" + timestamp.getMillis() + ":" + w + ":" + p;
+  }
+
   @Test
   @Category(NeedsRunner.class)
   public void testCreateTimestamped() {
@@ -321,6 +342,66 @@ public void testCreateTimestampedDefaultOutputCoderUsingTypeDescriptor() throws
     assertThat(p.apply(values).getCoder(), equalTo(coder));
   }
 
+  @Test
+  @Category(NeedsRunner.class)
+  public void testCreateWindowedValues() {
+    List<WindowedValue<String>> data =
+        Arrays.asList(
+            WindowedValue.of("a", new Instant(1L), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING),
+            WindowedValue.of("b", new Instant(2L), GlobalWindow.INSTANCE, PaneInfo.NO_FIRING),
+            WindowedValue.of(
+                "c", new Instant(3L), GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING));
+
+    // The easiest way to directly check the created PCollection with PAssert and without relying on
+    // other
+    // mechanisms than built-in DoFn processing is to dump it all to a string.
+    List<String> formattedData =
+        data.stream()
+            .flatMap(
+                (WindowedValue<String> windowedValue) ->
+                    windowedValue.getWindows().stream()
+                        .map(
+                            (BoundedWindow w) ->
+                                formatMetadata(
+                                    windowedValue.getValue(),
+                                    windowedValue.getTimestamp(),
+                                    w,
+                                    windowedValue.getPane())))
+            .collect(Collectors.toList());
+
+    PCollection<String> output =
+        p.apply(Create.windowedValues(data).withWindowCoder(GlobalWindow.Coder.INSTANCE))
+            .apply(ParDo.of(new FormatMetadata()));
+
+    PAssert.that(output).containsInAnyOrder(formattedData);
+    p.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void testCreateWindowedValuesEmpty() {
+    PCollection<String> output =
+        p.apply(
+            Create.windowedValues(new ArrayList<WindowedValue<String>>())
+                .withCoder(StringUtf8Coder.of()));
+
+    PAssert.that(output).empty();
+    p.run();
+  }
+
+  @Test
+  public void testCreateWindowedValuesEmptyUnspecifiedCoder() {
+    p.enableAbandonedNodeEnforcement(false);
+
+    thrown.expect(IllegalArgumentException.class);
+    thrown.expectMessage("determine a default Coder");
+    thrown.expectMessage("Create.empty(Coder)");
+    thrown.expectMessage("Create.empty(TypeDescriptor)");
+    thrown.expectMessage("withCoder(Coder)");
+    thrown.expectMessage("withType(TypeDescriptor)");
+    p.apply(Create.windowedValues(new ArrayList<>()));
+  }
+
   @Test
   @Category(ValidatesRunner.class)
   public void testCreateWithVoidType() throws Exception {
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
index 716bd8781831..e0e8a7feed6d 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/ReshuffleTest.java
@@ -24,13 +24,18 @@
 import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder;
 import static org.hamcrest.core.Is.is;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.Serializable;
 import java.util.List;
+import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.coders.VarIntCoder;
 import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.options.StreamingOptions;
+import org.apache.beam.sdk.runners.TransformHierarchy;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestStream;
@@ -40,12 +45,15 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Sessions;
 import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.util.WindowedValue;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TimestampedValue;
 import org.apache.beam.sdk.values.TypeDescriptors;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
 import org.apache.beam.sdk.values.WindowingStrategy;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists;
@@ -152,6 +160,81 @@ public void testReshufflePreservesTimestamps() {
     pipeline.run();
   }
 
+  /**
+   * Tests that window & pane info is preserved after applying a {@link Reshuffle} with the default
+   * {@link WindowingStrategy}.
+   */
+  @Test
+  @Category(ValidatesRunner.class)
+  public void testReshufflePreservesMetadata() {
+    PCollection<KV<String, ValueInSingleWindow<String>>> input =
+        pipeline
+            .apply(
+                Create.windowedValues(
+                        WindowedValue.of(
+                            "foo",
+                            BoundedWindow.TIMESTAMP_MIN_VALUE,
+                            GlobalWindow.INSTANCE,
+                            PaneInfo.NO_FIRING),
+                        WindowedValue.of(
+                            "foo",
+                            new Instant(0),
+                            GlobalWindow.INSTANCE,
+                            PaneInfo.ON_TIME_AND_ONLY_FIRING),
+                        WindowedValue.of(
+                            "bar",
+                            new Instant(33),
+                            GlobalWindow.INSTANCE,
+                            PaneInfo.createPane(false, false, PaneInfo.Timing.LATE, 1, 1)),
+                        WindowedValue.of(
+                            "bar",
+                            GlobalWindow.INSTANCE.maxTimestamp(),
+                            GlobalWindow.INSTANCE,
+                            PaneInfo.NO_FIRING))
+                    .withCoder(StringUtf8Coder.of())
+                    .withWindowCoder(GlobalWindow.Coder.INSTANCE))
+            .apply(WithKeys.<String, String>of(v -> v).withKeyType(TypeDescriptors.strings()))
+            .apply("ReifyOriginalMetadata", Reify.windowsInValue());
+
+    // The outer WindowedValue is the reified metadata post-reshuffle. The inner
+    // WindowedValue is the pre-reshuffle metadata.
+    PCollection<ValueInSingleWindow<ValueInSingleWindow<String>>> output =
+        input
+            .apply(Reshuffle.of())
+            .apply("ReifyReshuffledMetadata", Reify.windowsInValue())
+            .apply(Values.create());
+
+    PAssert.that(output)
+        .satisfies(
+            input1 -> {
+              for (ValueInSingleWindow<ValueInSingleWindow<String>> elem : input1) {
+                Instant originalTimestamp = elem.getValue().getTimestamp();
+                Instant afterReshuffleTimestamp = elem.getTimestamp();
+                assertThat(
+                    "Reshuffle did not preserve element timestamp for " + elem,
+                    afterReshuffleTimestamp,
+                    equalTo(originalTimestamp));
+
+                PaneInfo originalPaneInfo = elem.getValue().getPane();
+                PaneInfo afterReshufflePaneInfo = elem.getPane();
+                assertThat(
+                    "Reshuffle did not preserve pane info for " + elem,
+                    afterReshufflePaneInfo,
+                    equalTo(originalPaneInfo));
+
+                BoundedWindow originalWindow = elem.getValue().getWindow();
+                BoundedWindow afterReshuffleWindow = elem.getWindow();
+                assertThat(
+                    "Reshuffle did not preserve window for " + elem,
+                    afterReshuffleWindow,
+                    equalTo(originalWindow));
+              }
+              return null;
+            });
+
+    pipeline.run();
+  }
+
   @Test
   @Category(ValidatesRunner.class)
   public void testReshuffleAfterSessionsAndGroupByKey() {
@@ -301,4 +384,60 @@ public void testAssignShardFn() {
 
     pipeline.run();
   }
+
+  static class OldTransformSeeker extends Pipeline.PipelineVisitor.Defaults {
+    boolean isOldTransformFound = false;
+
+    // A class that is only found in the old expansion
+    private Class<?> restoreTimestampsClass = ReifyTimestamps.extractFromValues().getClass();
+
+    @Override
+    public CompositeBehavior enterCompositeTransform(TransformHierarchy.Node node) {
+      if (restoreTimestampsClass.isInstance(node.getTransform())) {
+        this.isOldTransformFound = true;
+        return CompositeBehavior.DO_NOT_ENTER_TRANSFORM;
+      } else {
+        return CompositeBehavior.ENTER_TRANSFORM;
+      }
+    }
+  }
+
+  @Test
+  public void testNoOldTransformByDefault() {
+    pipeline.apply(Create.of(KV.of("arbitrary", "kv"))).apply(Reshuffle.of());
+
+    OldTransformSeeker seeker = new OldTransformSeeker();
+    pipeline.traverseTopologically(seeker);
+    assertFalse(seeker.isOldTransformFound);
+  }
+
+  @Test
+  public void testRequestOldUpdateCompatibility() {
+    pipeline.getOptions().as(StreamingOptions.class).setUpdateCompatibilityVersion("2.53.0");
+    pipeline.apply(Create.of(KV.of("arbitrary", "kv"))).apply(Reshuffle.of());
+
+    OldTransformSeeker seeker = new OldTransformSeeker();
+    pipeline.traverseTopologically(seeker);
+    assertTrue(seeker.isOldTransformFound);
+  }
+
+  @Test
+  public void testRequestVeryOldUpdateCompatibility() {
+    pipeline.getOptions().as(StreamingOptions.class).setUpdateCompatibilityVersion("2.46.0");
+    pipeline.apply(Create.of(KV.of("arbitrary", "kv"))).apply(Reshuffle.of());
+
+    OldTransformSeeker seeker = new OldTransformSeeker();
+    pipeline.traverseTopologically(seeker);
+    assertTrue(seeker.isOldTransformFound);
+  }
+
+  @Test
+  public void testNoOldTransformInRecentVersion() {
+    pipeline.getOptions().as(StreamingOptions.class).setUpdateCompatibilityVersion("2.54.0");
+    pipeline.apply(Create.of(KV.of("arbitrary", "kv"))).apply(Reshuffle.of());
+
+    OldTransformSeeker seeker = new OldTransformSeeker();
+    pipeline.traverseTopologically(seeker);
+    assertFalse(seeker.isOldTransformFound);
+  }
 }
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/BRHEnabledPTransform.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/BRHEnabledPTransform.java
new file mode 100644
index 000000000000..b279ad938dba
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/BRHEnabledPTransform.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.DefaultErrorHandler;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+
+/**
+ * Dummy PTransform that is configurable with a Bad Record Handler. TODO(johncasey) look to factor
+ * some of this out for easy use in other IOs
+ */
+public class BRHEnabledPTransform extends PTransform<PCollection<Integer>, PCollection<Integer>> {
+
+  private ErrorHandler<BadRecord, ?> errorHandler = new DefaultErrorHandler<>();
+
+  private BadRecordRouter badRecordRouter = BadRecordRouter.THROWING_ROUTER;
+
+  private static final TupleTag<Integer> RECORDS = new TupleTag<>();
+
+  public BRHEnabledPTransform() {}
+
+  public BRHEnabledPTransform withBadRecordHandler(ErrorHandler<BadRecord, ?> errorHandler) {
+    this.errorHandler = errorHandler;
+    this.badRecordRouter = BadRecordRouter.RECORDING_ROUTER;
+    return this;
+  }
+
+  @Override
+  public PCollection<Integer> expand(PCollection<Integer> input) {
+    // TODO this pattern is a clunky. Look to improve this once we have ParDo level error handling.
+    PCollectionTuple pCollectionTuple =
+        input.apply(
+            "NoOpDoFn",
+            ParDo.of(new OddIsBad(badRecordRouter))
+                .withOutputTags(RECORDS, TupleTagList.of(BadRecordRouter.BAD_RECORD_TAG)));
+
+    errorHandler.addErrorCollection(
+        pCollectionTuple
+            .get(BadRecordRouter.BAD_RECORD_TAG)
+            .setCoder(BadRecord.getCoder(input.getPipeline())));
+
+    return pCollectionTuple.get(RECORDS).setCoder(BigEndianIntegerCoder.of());
+  }
+
+  public static class OddIsBad extends DoFn<Integer, Integer> {
+
+    private final BadRecordRouter badRecordRouter;
+
+    public OddIsBad(BadRecordRouter badRecordRouter) {
+      this.badRecordRouter = badRecordRouter;
+    }
+
+    @ProcessElement
+    public void processElement(@Element Integer element, MultiOutputReceiver receiver)
+        throws Exception {
+      if (element % 2 == 0) {
+        receiver.get(RECORDS).output(element);
+      } else {
+        badRecordRouter.route(
+            receiver,
+            element,
+            BigEndianIntegerCoder.of(),
+            new RuntimeException("Integer was odd"),
+            "Integer was odd");
+      }
+    }
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/BadRecordRouterTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/BadRecordRouterTest.java
new file mode 100644
index 000000000000..fb18a6077db1
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/BadRecordRouterTest.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import static org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter.BAD_RECORD_TAG;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.function.BiFunction;
+import org.apache.beam.sdk.coders.BigEndianIntegerCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.CoderException;
+import org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord.Record;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentMatcher;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+
+@RunWith(JUnit4.class)
+public class BadRecordRouterTest {
+
+  @Rule public final MockitoRule mockito = MockitoJUnit.rule();
+
+  @Rule public ExpectedException thrown = ExpectedException.none();
+
+  @Mock private MultiOutputReceiver outputReceiver;
+
+  @Mock private OutputReceiver<BadRecord> badRecordOutputReceiver;
+
+  private static final BiFunction<
+          BadRecord.Builder, BadRecord.Failure.Builder, ArgumentMatcher<BadRecord>>
+      ignoreStacktraceMatcher =
+          (expectedBuilder, failure) ->
+              (ArgumentMatcher<BadRecord>)
+                  argument -> {
+                    // This complex matcher means we don't need to maintain an expected stacktrace
+                    String stackTrace = argument.getFailure().getExceptionStacktrace();
+                    failure.setExceptionStacktrace(stackTrace);
+                    BadRecord expected = expectedBuilder.setFailure(failure.build()).build();
+                    return expected.equals(argument);
+                  };
+
+  @Test
+  public void testThrowingHandlerWithException() throws Exception {
+    BadRecordRouter handler = BadRecordRouter.THROWING_ROUTER;
+
+    thrown.expect(IOException.class);
+
+    handler.route(outputReceiver, new Object(), null, new IOException(), "desc");
+  }
+
+  @Test
+  public void testThrowingHandlerWithNoException() throws Exception {
+    BadRecordRouter handler = BadRecordRouter.THROWING_ROUTER;
+
+    thrown.expect(RuntimeException.class);
+
+    handler.route(outputReceiver, new Object(), null, null, "desc");
+  }
+
+  @Test
+  public void testRecordingHandler() throws Exception {
+    when(outputReceiver.get(BAD_RECORD_TAG)).thenReturn(badRecordOutputReceiver);
+
+    BadRecordRouter handler = BadRecordRouter.RECORDING_ROUTER;
+
+    handler.route(outputReceiver, 5, BigEndianIntegerCoder.of(), new RuntimeException(), "desc");
+
+    BadRecord.Builder expectedBuilder =
+        BadRecord.builder()
+            .setRecord(
+                Record.builder()
+                    .setHumanReadableJsonRecord("5")
+                    .setEncodedRecord(new byte[] {0, 0, 0, 5})
+                    .setCoder("BigEndianIntegerCoder")
+                    .build());
+
+    BadRecord.Failure.Builder failure =
+        BadRecord.Failure.builder()
+            .setException("java.lang.RuntimeException")
+            .setDescription("desc");
+
+    verify(badRecordOutputReceiver)
+        .output(ArgumentMatchers.argThat(ignoreStacktraceMatcher.apply(expectedBuilder, failure)));
+  }
+
+  @Test
+  public void testNoCoder() throws Exception {
+    when(outputReceiver.get(BAD_RECORD_TAG)).thenReturn(badRecordOutputReceiver);
+
+    BadRecordRouter handler = BadRecordRouter.RECORDING_ROUTER;
+
+    handler.route(outputReceiver, 5, null, new RuntimeException(), "desc");
+
+    BadRecord.Builder expectedBuilder =
+        BadRecord.builder().setRecord(Record.builder().setHumanReadableJsonRecord("5").build());
+
+    BadRecord.Failure.Builder failure =
+        BadRecord.Failure.builder()
+            .setException("java.lang.RuntimeException")
+            .setDescription("desc");
+
+    verify(badRecordOutputReceiver)
+        .output(ArgumentMatchers.argThat(ignoreStacktraceMatcher.apply(expectedBuilder, failure)));
+  }
+
+  @Test
+  public void testFailingCoder() throws Exception {
+    when(outputReceiver.get(BAD_RECORD_TAG)).thenReturn(badRecordOutputReceiver);
+
+    BadRecordRouter handler = BadRecordRouter.RECORDING_ROUTER;
+
+    Coder<Integer> failingCoder =
+        new Coder<Integer>() {
+          @Override
+          public void encode(Integer value, OutputStream outStream)
+              throws CoderException, IOException {
+            throw new CoderException("Failing Coder");
+          }
+
+          @Override
+          public Integer decode(InputStream inStream) throws CoderException, IOException {
+            return null;
+          }
+
+          @Override
+          public List<? extends Coder<?>> getCoderArguments() {
+            return null;
+          }
+
+          @Override
+          public void verifyDeterministic() throws NonDeterministicException {}
+        };
+
+    handler.route(outputReceiver, 5, failingCoder, new RuntimeException(), "desc");
+
+    BadRecord.Builder expectedBuilder =
+        BadRecord.builder()
+            .setRecord(
+                Record.builder()
+                    .setHumanReadableJsonRecord("5")
+                    .setCoder(failingCoder.toString())
+                    .build());
+
+    BadRecord.Failure.Builder failure =
+        BadRecord.Failure.builder()
+            .setException("java.lang.RuntimeException")
+            .setDescription("desc");
+
+    verify(badRecordOutputReceiver)
+        .output(ArgumentMatchers.argThat(ignoreStacktraceMatcher.apply(expectedBuilder, failure)));
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandlerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandlerTest.java
new file mode 100644
index 000000000000..b0a5733cb1bd
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandlerTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import java.util.Objects;
+import org.apache.beam.sdk.testing.NeedsRunner;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord.Record;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
+import org.apache.beam.sdk.values.PCollection;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class ErrorHandlerTest {
+  @Rule public final TestPipeline pipeline = TestPipeline.create();
+  @Rule public ExpectedException thrown = ExpectedException.none();
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void testNoUsageErrorHandlerUsage() throws Exception {
+    try (BadRecordErrorHandler<PCollection<BadRecord>> eh =
+        pipeline.registerBadRecordErrorHandler(new DummySinkTransform<>())) {}
+
+    pipeline.run();
+  }
+
+  @Test
+  public void testUnclosedErrorHandlerUsage() {
+
+    pipeline.registerBadRecordErrorHandler(new DummySinkTransform<>());
+
+    // Expected to be thrown because the error handler isn't closed
+    thrown.expect(IllegalStateException.class);
+
+    pipeline.run();
+  }
+
+  @Test
+  public void testBRHEnabledPTransform() {
+    PCollection<Integer> record = pipeline.apply(Create.of(1, 2, 3, 4));
+    record.apply(new BRHEnabledPTransform());
+
+    // unhandled runtime exception thrown by the BRHEnabledPTransform
+    thrown.expect(RuntimeException.class);
+
+    pipeline.run();
+  }
+
+  @Test
+  @Category(NeedsRunner.class)
+  public void testErrorHandlerWithBRHTransform() throws Exception {
+    PCollection<Integer> record = pipeline.apply(Create.of(1, 2, 3, 4));
+    DummySinkTransform<BadRecord> transform = new DummySinkTransform<>();
+    ErrorHandler<BadRecord, PCollection<BadRecord>> eh =
+        pipeline.registerBadRecordErrorHandler(transform);
+    record.apply(new BRHEnabledPTransform().withBadRecordHandler(eh));
+    eh.close();
+    PCollection<BadRecord> badRecords = eh.getOutput();
+
+    // We use a more complex satisfies statement to ensure we don't need to preserve stacktraces
+    // in test cases
+    PAssert.that(badRecords)
+        .satisfies(
+            (records) -> {
+              int count = 0;
+              for (BadRecord badRecord : records) {
+                count++;
+
+                Record r = null;
+
+                if (Objects.equals(badRecord.getRecord().getHumanReadableJsonRecord(), "1")) {
+                  r =
+                      Record.builder()
+                          .setHumanReadableJsonRecord("1")
+                          .setEncodedRecord(new byte[] {0, 0, 0, 1})
+                          .setCoder("BigEndianIntegerCoder")
+                          .build();
+                } else {
+                  r =
+                      Record.builder()
+                          .setHumanReadableJsonRecord("3")
+                          .setEncodedRecord(new byte[] {0, 0, 0, 3})
+                          .setCoder("BigEndianIntegerCoder")
+                          .build();
+                }
+
+                BadRecord.Builder expectedBuilder = BadRecord.builder().setRecord(r);
+
+                BadRecord.Failure.Builder failure =
+                    BadRecord.Failure.builder()
+                        .setException("java.lang.RuntimeException: Integer was odd")
+                        .setDescription("Integer was odd");
+
+                failure.setExceptionStacktrace(badRecord.getFailure().getExceptionStacktrace());
+                expectedBuilder.setFailure(failure.build());
+                Assert.assertEquals("Expect failure to match", expectedBuilder.build(), badRecord);
+              }
+              Assert.assertEquals("Expect 2 errors", 2, count);
+              return null;
+            });
+
+    pipeline.run().waitUntilFinish();
+  }
+
+  public static class DummySinkTransform<T> extends PTransform<PCollection<T>, PCollection<T>> {
+
+    @Override
+    public PCollection<T> expand(PCollection<T> input) {
+      return input;
+    }
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandlingTestUtils.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandlingTestUtils.java
new file mode 100644
index 000000000000..41367765b920
--- /dev/null
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/errorhandling/ErrorHandlingTestUtils.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.transforms.errorhandling;
+
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.windowing.FixedWindows;
+import org.apache.beam.sdk.transforms.windowing.Window;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollection.IsBounded;
+import org.checkerframework.checker.initialization.qual.Initialized;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.UnknownKeyFor;
+import org.joda.time.Duration;
+
+public class ErrorHandlingTestUtils {
+  public static class ErrorSinkTransform
+      extends PTransform<PCollection<BadRecord>, PCollection<Long>> {
+
+    @Override
+    public @UnknownKeyFor @NonNull @Initialized PCollection<Long> expand(
+        PCollection<BadRecord> input) {
+      if (input.isBounded() == IsBounded.BOUNDED) {
+        return input.apply("Combine", Combine.globally(Count.<BadRecord>combineFn()));
+      } else {
+        return input
+            .apply("Window", Window.into(FixedWindows.of(Duration.standardDays(1))))
+            .apply("Combine", Combine.globally(Count.<BadRecord>combineFn()).withoutDefaults());
+      }
+    }
+  }
+}
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
index a99d18a3d91f..bcd1f9ccf7b5 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/reflect/DoFnInvokersTest.java
@@ -41,6 +41,7 @@
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import org.apache.beam.sdk.coders.AtomicCoder;
 import org.apache.beam.sdk.coders.CoderException;
@@ -75,6 +76,7 @@
 import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.IntervalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.joda.time.Instant;
 import org.junit.Before;
@@ -556,6 +558,15 @@ public void output(SomeRestriction output) {
               public void outputWithTimestamp(SomeRestriction output, Instant timestamp) {
                 fail("Unexpected output with timestamp");
               }
+
+              @Override
+              public void outputWindowedValue(
+                  SomeRestriction output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                fail("Unexpected outputWindowedValue");
+              }
             };
           }
         });
@@ -800,6 +811,17 @@ public void outputWithTimestamp(String output, Instant instant) {
                 invoked = true;
                 assertEquals("foo", output);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  String output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                assertFalse(invoked);
+                invoked = true;
+                assertEquals("foo", output);
+              }
             };
           }
         });
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/HistogramDataTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/HistogramDataTest.java
index 8716bbc1411f..42d02541540c 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/util/HistogramDataTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/util/HistogramDataTest.java
@@ -354,4 +354,77 @@ public void testStatistics_sumOfSquaredDeviations() {
 
     assertThat(histogram.getSumOfSquaredDeviations(), equalTo(8250.0));
   }
+
+  @Test
+  public void testGetAndReset_resetSucceeds() {
+    // Records values from [20, 50) in three buckets that have width 10.
+    HistogramData originalHistogram = HistogramData.linear(20, 10, 3);
+    originalHistogram.record(15.0, 25.0, 35.0, 45.0, 55.0);
+    originalHistogram.getAndReset();
+
+    HistogramData emptyHistogramData = HistogramData.linear(20, 10, 3);
+    assertThat(originalHistogram, equalTo(emptyHistogramData));
+    assertThat(originalHistogram.getMean(), equalTo(0.0));
+    assertThat(originalHistogram.getSumOfSquaredDeviations(), equalTo(0.0));
+    assertThat(originalHistogram.getTopBucketMean(), equalTo(0.0));
+    assertThat(originalHistogram.getBottomBucketMean(), equalTo(0.0));
+  }
+
+  @Test
+  public void testGetAndReset_getSucceeds() {
+    // Records values from [20, 50) in three buckets that have width 10.
+    HistogramData originalHistogram = HistogramData.linear(20, 10, 3);
+    originalHistogram.record(15.0, 25.0, 35.0, 45.0, 55.0);
+    HistogramData copyHistogram = originalHistogram.getAndReset();
+
+    HistogramData duplicateHistogram = HistogramData.linear(20, 10, 3);
+    duplicateHistogram.record(15.0, 25.0, 35.0, 45.0, 55.0);
+    assertThat(copyHistogram, equalTo(duplicateHistogram));
+    assertThat(copyHistogram.getBucketType(), equalTo(originalHistogram.getBucketType()));
+    assertThat(copyHistogram.getMean(), equalTo(35.0));
+    assertThat(copyHistogram.getSumOfSquaredDeviations(), equalTo(1000.0));
+    assertThat(copyHistogram.getTopBucketMean(), equalTo(55.0));
+    assertThat(copyHistogram.getBottomBucketMean(), equalTo(15.0));
+  }
+
+  @Test
+  public void recordUnderflowValue() {
+    // 'histogram' to record values from [0, 32). Values outside this range are
+    // recorded in the overflow/underflow bin.
+    HistogramData histogram = HistogramData.exponential(0, 5);
+
+    assertThat(histogram.getTopBucketCount(), equalTo(0L));
+    assertThat(histogram.getTopBucketMean(), equalTo(0.0));
+
+    histogram.record(32);
+    assertThat(histogram.getTopBucketCount(), equalTo(1L));
+    assertThat(histogram.getTopBucketMean(), equalTo(32.0));
+
+    histogram.record(40.0, 48.0, 56.0);
+    assertThat(histogram.getTopBucketCount(), equalTo(4L));
+    assertThat(histogram.getTopBucketMean(), equalTo(44.0));
+  }
+
+  @Test
+  public void recordOverflowValue() {
+    // 'histogram' to record values from [50, 150). Values outside this range are
+    // recorded in the overflow/underflow bin.
+    HistogramData histogram = HistogramData.linear(50, 10, 10);
+
+    assertThat(histogram.getBottomBucketCount(), equalTo(0L));
+    assertThat(histogram.getBottomBucketMean(), equalTo(0.0));
+
+    histogram.record(-20);
+    assertThat(histogram.getBottomBucketCount(), equalTo(1L));
+    assertThat(histogram.getBottomBucketMean(), equalTo(-20.0));
+
+    histogram.record(-30.0, -40.0);
+    assertThat(histogram.getBottomBucketCount(), equalTo(3L));
+    assertThat(histogram.getBottomBucketMean(), equalTo(-30.0));
+
+    histogram.clear();
+    histogram.record(25.0, 40.0);
+    assertThat(histogram.getBottomBucketCount(), equalTo(2L));
+    assertThat(histogram.getBottomBucketMean(), equalTo(32.5));
+  }
 }
diff --git a/sdks/java/expansion-service/build.gradle b/sdks/java/expansion-service/build.gradle
index 7bc77e3aea80..99c515cd4e63 100644
--- a/sdks/java/expansion-service/build.gradle
+++ b/sdks/java/expansion-service/build.gradle
@@ -36,11 +36,14 @@ test {
 
 dependencies {
   implementation project(path: ":model:pipeline", configuration: "shadow")
+  implementation project(path: ":model:fn-execution", configuration: "shadow")
   implementation project(path: ":model:job-management", configuration: "shadow")
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation project(path: ":runners:core-construction-java")
   implementation project(path: ":runners:java-fn-execution")
   implementation project(path: ":sdks:java:fn-execution")
+  implementation project(path: ":sdks:java:harness")
+  permitUnusedDeclared project(path: ":model:fn-execution")
   permitUnusedDeclared project(path: ":sdks:java:fn-execution")
   implementation library.java.jackson_annotations
   implementation library.java.jackson_databind
diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java
index c6509a5bbb4a..7760cab64acc 100644
--- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java
+++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionService.java
@@ -30,12 +30,14 @@
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.stream.Collectors;
+import org.apache.beam.fn.harness.ExternalWorkerService;
 import org.apache.beam.model.expansion.v1.ExpansionApi;
 import org.apache.beam.model.expansion.v1.ExpansionApi.DiscoverSchemaTransformRequest;
 import org.apache.beam.model.expansion.v1.ExpansionApi.DiscoverSchemaTransformResponse;
@@ -48,6 +50,7 @@
 import org.apache.beam.model.pipeline.v1.SchemaApi;
 import org.apache.beam.runners.core.construction.Environments;
 import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;
 import org.apache.beam.runners.core.construction.PipelineTranslation;
 import org.apache.beam.runners.core.construction.RehydratedComponents;
 import org.apache.beam.runners.core.construction.SdkComponents;
@@ -65,6 +68,7 @@
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PortablePipelineOptions;
+import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.schemas.NoSuchSchemaException;
 import org.apache.beam.sdk.schemas.Schema;
 import org.apache.beam.sdk.schemas.Schema.Field;
@@ -81,6 +85,7 @@
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.PInput;
 import org.apache.beam.sdk.values.POutput;
+import org.apache.beam.sdk.values.PValue;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString;
@@ -456,6 +461,22 @@ default Map<String, PCollection<?>> extractOutputs(OutputT output) {
           i++;
         }
         return indexToPCollection.build();
+      } else if (output instanceof POutput) {
+        // This is needed to support custom output types.
+        Map<TupleTag<?>, PValue> values = output.expand();
+        Map<String, PCollection<?>> returnMap = new HashMap<>();
+        for (Map.Entry<TupleTag<?>, PValue> entry : values.entrySet()) {
+          if (!(entry.getValue() instanceof PCollection)) {
+            throw new UnsupportedOperationException(
+                "Unable to parse the output type "
+                    + output.getClass()
+                    + " due to key "
+                    + entry.getKey()
+                    + " not mapping to a PCollection");
+          }
+          returnMap.put(entry.getKey().getId(), (PCollection<?>) entry.getValue());
+        }
+        return returnMap;
       } else {
         throw new UnsupportedOperationException("Unknown output type: " + output.getClass());
       }
@@ -514,6 +535,7 @@ default List<String> getDependencies(RunnerApi.FunctionSpec spec, PipelineOption
 
   private @MonotonicNonNull Map<String, TransformProvider> registeredTransforms;
   private final PipelineOptions pipelineOptions;
+  private final @Nullable String loopbackAddress;
 
   public ExpansionService() {
     this(new String[] {});
@@ -524,7 +546,12 @@ public ExpansionService(String[] args) {
   }
 
   public ExpansionService(PipelineOptions opts) {
+    this(opts, null);
+  }
+
+  public ExpansionService(PipelineOptions opts, @Nullable String loopbackAddress) {
     this.pipelineOptions = opts;
+    this.loopbackAddress = loopbackAddress;
   }
 
   private Map<String, TransformProvider> getRegisteredTransforms() {
@@ -559,7 +586,8 @@ private Map<String, TransformProvider> loadRegisteredTransforms() {
         request.getTransform().getSpec().getUrn());
     LOG.debug("Full transform: {}", request.getTransform());
     Set<String> existingTransformIds = request.getComponents().getTransformsMap().keySet();
-    Pipeline pipeline = createPipeline();
+    Pipeline pipeline =
+        createPipeline(PipelineOptionsTranslation.fromProto(request.getPipelineOptions()));
     boolean isUseDeprecatedRead =
         ExperimentalOptions.hasExperiment(pipelineOptions, "use_deprecated_read")
             || ExperimentalOptions.hasExperiment(
@@ -628,9 +656,19 @@ private Map<String, TransformProvider> loadRegisteredTransforms() {
         rehydratedComponents
             .getSdkComponents(request.getRequirementsList())
             .withNewIdPrefix(request.getNamespace());
-    sdkComponents.registerEnvironment(
+    RunnerApi.Environment defaultEnvironment =
         Environments.createOrGetDefaultEnvironment(
-            pipeline.getOptions().as(PortablePipelineOptions.class)));
+            pipeline.getOptions().as(PortablePipelineOptions.class));
+    if (pipelineOptions.as(ExpansionServiceOptions.class).getAlsoStartLoopbackWorker()) {
+      PortablePipelineOptions externalOptions =
+          PipelineOptionsFactory.create().as(PortablePipelineOptions.class);
+      externalOptions.setDefaultEnvironmentType(Environments.ENVIRONMENT_EXTERNAL);
+      externalOptions.setDefaultEnvironmentConfig(loopbackAddress);
+      defaultEnvironment =
+          Environments.createAnyOfEnvironment(
+              defaultEnvironment, Environments.createOrGetDefaultEnvironment(externalOptions));
+    }
+    sdkComponents.registerEnvironment(defaultEnvironment);
     Map<String, String> outputMap =
         outputs.entrySet().stream()
             .collect(
@@ -672,7 +710,7 @@ private Map<String, TransformProvider> loadRegisteredTransforms() {
         .build();
   }
 
-  protected Pipeline createPipeline() {
+  protected Pipeline createPipeline(PipelineOptions requestOptions) {
     // TODO: [https://github.com/apache/beam/issues/21064]: implement proper validation
     PipelineOptions effectiveOpts = PipelineOptionsFactory.create();
     PortablePipelineOptions portableOptions = effectiveOpts.as(PortablePipelineOptions.class);
@@ -693,6 +731,14 @@ protected Pipeline createPipeline() {
         .as(ExpansionServiceOptions.class)
         .setExpansionServiceConfig(
             pipelineOptions.as(ExpansionServiceOptions.class).getExpansionServiceConfig());
+    // TODO(https://github.com/apache/beam/issues/20090): Figure out the correct subset of options
+    // to propagate.
+    if (requestOptions.as(StreamingOptions.class).getUpdateCompatibilityVersion() != null) {
+      effectiveOpts
+          .as(StreamingOptions.class)
+          .setUpdateCompatibilityVersion(
+              requestOptions.as(StreamingOptions.class).getUpdateCompatibilityVersion());
+    }
     return Pipeline.create(effectiveOpts);
   }
 
@@ -721,6 +767,7 @@ DiscoverSchemaTransformResponse discover(DiscoverSchemaTransformRequest request)
         transformProvider.getAllProviders()) {
       SchemaTransformConfig.Builder schemaTransformConfigBuilder =
           SchemaTransformConfig.newBuilder();
+      schemaTransformConfigBuilder.setDescription(provider.description());
       schemaTransformConfigBuilder.setConfigSchema(
           SchemaTranslation.schemaToProto(provider.configurationSchema(), true));
       schemaTransformConfigBuilder.addAllInputPcollectionNames(provider.inputCollectionNames());
@@ -759,9 +806,12 @@ public static void main(String[] args) throws Exception {
 
     // Register the options class used by the expansion service.
     PipelineOptionsFactory.register(ExpansionServiceOptions.class);
+    @SuppressWarnings({"nullness"})
+    PipelineOptions options =
+        PipelineOptionsFactory.fromArgs(Arrays.copyOfRange(args, 1, args.length)).create();
 
     @SuppressWarnings("nullness")
-    ExpansionService service = new ExpansionService(Arrays.copyOfRange(args, 1, args.length));
+    ExpansionService service = new ExpansionService(options, "localhost:" + port);
 
     StringBuilder registeredTransformsLog = new StringBuilder();
     boolean registeredTransformsFound = false;
@@ -794,11 +844,12 @@ public static void main(String[] args) throws Exception {
       System.out.println("\nDid not find any registered transforms or SchemaTransforms.\n");
     }
 
-    Server server =
-        ServerBuilder.forPort(port)
-            .addService(service)
-            .addService(new ArtifactRetrievalService())
-            .build();
+    ServerBuilder serverBuilder =
+        ServerBuilder.forPort(port).addService(service).addService(new ArtifactRetrievalService());
+    if (options.as(ExpansionServiceOptions.class).getAlsoStartLoopbackWorker()) {
+      serverBuilder.addService(new ExternalWorkerService(options));
+    }
+    Server server = serverBuilder.build();
     server.start();
     server.awaitTermination();
   }
diff --git a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceOptions.java b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceOptions.java
index b3af511cd94f..6b569c7fce38 100644
--- a/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceOptions.java
+++ b/sdks/java/expansion-service/src/main/java/org/apache/beam/sdk/expansion/service/ExpansionServiceOptions.java
@@ -42,6 +42,11 @@ public interface ExpansionServiceOptions extends PipelineOptions {
 
   void setJavaClassLookupAllowlistFile(String file);
 
+  @Description("Whether to also start a loopback worker as part of this service.")
+  boolean getAlsoStartLoopbackWorker();
+
+  void setAlsoStartLoopbackWorker(boolean value);
+
   @Description("Expansion service configuration file.")
   String getExpansionServiceConfigFile();
 
diff --git a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServerTest.java b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServerTest.java
index 03938a073c4f..5d0273964db4 100644
--- a/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServerTest.java
+++ b/sdks/java/expansion-service/src/test/java/org/apache/beam/sdk/expansion/service/ExpansionServerTest.java
@@ -23,6 +23,7 @@
 import static org.hamcrest.core.Is.is;
 
 import java.util.Arrays;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.PortablePipelineOptions;
 import org.junit.Test;
 
@@ -58,7 +59,7 @@ public void testPassingPipelineArguments() {
     ExpansionService service = new ExpansionService(args);
     assertThat(
         service
-            .createPipeline()
+            .createPipeline(PipelineOptionsFactory.create())
             .getOptions()
             .as(PortablePipelineOptions.class)
             .getDefaultEnvironmentType(),
@@ -70,7 +71,11 @@ public void testNonEmptyFilesToStage() {
     String[] args = {"--filesToStage=nonExistent1.jar,nonExistent2.jar"};
     ExpansionService service = new ExpansionService(args);
     assertThat(
-        service.createPipeline().getOptions().as(PortablePipelineOptions.class).getFilesToStage(),
+        service
+            .createPipeline(PipelineOptionsFactory.create())
+            .getOptions()
+            .as(PortablePipelineOptions.class)
+            .getFilesToStage(),
         equalTo(Arrays.asList("nonExistent1.jar", "nonExistent2.jar")));
   }
 
@@ -79,7 +84,11 @@ public void testEmptyFilesToStageIsOK() {
     String[] args = {"--filesToStage="};
     ExpansionService service = new ExpansionService(args);
     assertThat(
-        service.createPipeline().getOptions().as(PortablePipelineOptions.class).getFilesToStage(),
+        service
+            .createPipeline(PipelineOptionsFactory.create())
+            .getOptions()
+            .as(PortablePipelineOptions.class)
+            .getFilesToStage(),
         equalTo(Arrays.asList("")));
   }
 }
diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java
index a65db5a90bad..2e4939560ad1 100644
--- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java
+++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java
@@ -69,6 +69,8 @@
 import org.apache.beam.sdk.transforms.SerializableFunctions;
 import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
@@ -337,6 +339,10 @@
  * events.apply("WriteAvros", AvroIO.<Integer>writeCustomTypeToGenericRecords()
  *     .to(new UserDynamicAvroDestinations(userToSchemaMap)));
  * }</pre>
+ *
+ * <p>Error handling for writing records that are malformed can be handled by using {@link
+ * TypedWrite#withBadRecordErrorHandler(ErrorHandler)}. See documentation in {@link FileIO} for
+ * details on usage
  */
 @SuppressWarnings({
   "nullness" // TODO(https://github.com/apache/beam/issues/20497)
@@ -1427,6 +1433,8 @@ public abstract static class TypedWrite<UserT, DestinationT, OutputT>
 
     abstract AvroSink.@Nullable DatumWriterFactory<OutputT> getDatumWriterFactory();
 
+    abstract @Nullable ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
     /**
      * The codec used to encode the blocks in the Avro file. String value drawn from those in
      * https://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/file/CodecFactory.html
@@ -1489,6 +1497,9 @@ abstract Builder<UserT, DestinationT, OutputT> setDynamicDestinations(
       abstract Builder<UserT, DestinationT, OutputT> setDatumWriterFactory(
           AvroSink.DatumWriterFactory<OutputT> datumWriterFactory);
 
+      abstract Builder<UserT, DestinationT, OutputT> setBadRecordErrorHandler(
+          @Nullable ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
       abstract TypedWrite<UserT, DestinationT, OutputT> build();
     }
 
@@ -1713,6 +1724,12 @@ public TypedWrite<UserT, DestinationT, OutputT> withMetadata(Map<String, Object>
       return toBuilder().setMetadata(ImmutableMap.copyOf(metadata)).build();
     }
 
+    /** See {@link FileIO.Write#withBadRecordErrorHandler(ErrorHandler)} for details on usage. */
+    public TypedWrite<UserT, DestinationT, OutputT> withBadRecordErrorHandler(
+        ErrorHandler<BadRecord, ?> errorHandler) {
+      return toBuilder().setBadRecordErrorHandler(errorHandler).build();
+    }
+
     DynamicAvroDestinations<UserT, DestinationT, OutputT> resolveDynamicDestinations() {
       DynamicAvroDestinations<UserT, DestinationT, OutputT> dynamicDestinations =
           getDynamicDestinations();
@@ -1782,6 +1799,9 @@ public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
       if (getNoSpilling()) {
         write = write.withNoSpilling();
       }
+      if (getBadRecordErrorHandler() != null) {
+        write = write.withBadRecordErrorHandler(getBadRecordErrorHandler());
+      }
       return input.apply("Write", write);
     }
 
diff --git a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java
index 08b9f5456dd2..bc9853beca46 100644
--- a/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java
+++ b/sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilIT.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.beam.sdk.extensions.gcp.options.GcsOptions;
 import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.testing.TestPipelineOptions;
 import org.apache.beam.sdk.testing.UsesKms;
@@ -53,7 +54,8 @@ public void testRewriteMultiPart() throws IOException {
     // Using a KMS key is necessary to trigger multi-part rewrites (bucket is created
     // with a bucket default key).
     assertNotNull(options.getTempRoot());
-    options.setTempLocation(options.getTempRoot() + "/testRewriteMultiPart");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "testRewriteMultiPart").toString());
 
     GcsOptions gcsOptions = options.as(GcsOptions.class);
     GcsUtil gcsUtil = gcsOptions.getGcsUtil();
diff --git a/sdks/java/extensions/sql/build.gradle b/sdks/java/extensions/sql/build.gradle
index 8a22bff69b7f..6f34891c2d3f 100644
--- a/sdks/java/extensions/sql/build.gradle
+++ b/sdks/java/extensions/sql/build.gradle
@@ -240,6 +240,17 @@ task integrationTest(type: Test) {
   useJUnit { }
 }
 
+task preCommit {
+  dependsOn build
+  dependsOn runBasicExample
+  dependsOn runPojoExample
+
+  if (project.hasProperty("testJavaVersion")) {
+    var testVer = project.property("testJavaVersion")
+    dependsOn(":sdks:java:testing:test-utils:verifyJavaVersion$testVer")
+  }
+}
+
 task postCommit {
   group = "Verification"
   description = "Various integration tests"
diff --git a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java
index 54415644152f..f032da0799d8 100644
--- a/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java
+++ b/sdks/java/extensions/sql/expansion-service/src/main/java/org/apache/beam/sdk/extensions/sql/expansion/SqlTransformSchemaTransformProvider.java
@@ -71,6 +71,21 @@ public String identifier() {
     return "schematransform:org.apache.beam:sql_transform:v1";
   }
 
+  @Override
+  public String description() {
+    return "A transform that executes a SQL query on its input PCollections.\n\n"
+        + "If a single input is given, it may be referred to as `PCOLLECTION`, e.g. the query could be of the form"
+        + "\n\n"
+        + "    SELECT a, sum(b) FROM PCOLLECTION"
+        + "\n\n"
+        + "If multiple inputs are given, the should be named as they are in the query, e.g."
+        + "\n\n"
+        + "    SELECT a, b, c FROM pcoll_1 join pcoll_2 using (b)"
+        + "\n\n"
+        + "For more details about Beam SQL in general see "
+        + "[the Beam SQL documentation](https://beam.apache.org/documentation/dsls/sql/overview/).";
+  }
+
   @Override
   public Schema configurationSchema() {
     List<String> providers = new ArrayList<>();
@@ -82,7 +97,7 @@ public Schema configurationSchema() {
     EnumerationType providerEnum = EnumerationType.create(providers);
 
     return Schema.of(
-        Schema.Field.of("query", Schema.FieldType.STRING),
+        Schema.Field.of("query", Schema.FieldType.STRING).withDescription("SQL query to execute"),
         Schema.Field.nullable(
             "ddl", Schema.FieldType.STRING), // TODO: Underlying builder seems more capable?
         Schema.Field.nullable("dialect", Schema.FieldType.logicalType(QUERY_ENUMERATION)),
diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java
index d60ebe46b370..ad856c1c3a7d 100644
--- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java
+++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/BeamZetaSqlCalcRel.java
@@ -26,6 +26,7 @@
 import com.google.zetasql.Value;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.util.ArrayDeque;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -48,6 +49,7 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
 import org.apache.beam.sdk.values.PCollectionTuple;
@@ -361,6 +363,16 @@ public void output(Row output) {
       public void outputWithTimestamp(Row output, Instant timestamp) {
         c.output(tag, output, timestamp, w);
       }
+
+      @Override
+      public void outputWindowedValue(
+          Row output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        throw new UnsupportedOperationException(
+            "outputWindowedValue not supported in finish bundle here");
+      }
     }
 
     private static RuntimeException extractException(Throwable e) {
diff --git a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/JvmInitializers.java b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/JvmInitializers.java
index c12adf8c02f1..f739a797af80 100644
--- a/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/JvmInitializers.java
+++ b/sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/JvmInitializers.java
@@ -35,6 +35,7 @@ public static void runOnStartup() {
       // We write to standard out since logging has yet to be initialized.
       System.out.format("Running JvmInitializer#onStartup for %s%n", initializer);
       initializer.onStartup();
+      System.out.format("Completed JvmInitializer#onStartup for %s%n", initializer);
     }
   }
 
@@ -52,6 +53,7 @@ public static void runBeforeProcessing(PipelineOptions options) {
     for (JvmInitializer initializer : ReflectHelpers.loadServicesOrdered(JvmInitializer.class)) {
       logger.info("Running JvmInitializer#beforeProcessing for {}", initializer);
       initializer.beforeProcessing(options);
+      logger.info("Completed JvmInitializer#beforeProcessing for {}", initializer);
     }
   }
 }
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
index ddf52125b2e4..69114703a528 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/FnApiDoFnRunner.java
@@ -33,6 +33,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
@@ -168,6 +169,7 @@ static class Factory<InputT, RestrictionT, PositionT, WatermarkEstimatorStateT,
       FnApiDoFnRunner<InputT, RestrictionT, PositionT, WatermarkEstimatorStateT, OutputT> runner =
           new FnApiDoFnRunner<>(
               context.getPipelineOptions(),
+              context.getRunnerCapabilities(),
               context.getShortIdMap(),
               context.getBeamFnStateClient(),
               context.getPTransformId(),
@@ -336,6 +338,7 @@ static class Factory<InputT, RestrictionT, PositionT, WatermarkEstimatorStateT,
 
   FnApiDoFnRunner(
       PipelineOptions pipelineOptions,
+      Set<String> runnerCapabilities,
       ShortIdMap shortIds,
       BeamFnStateClient beamFnStateClient,
       String pTransformId,
@@ -740,6 +743,7 @@ private ByteString encodeProgress(double value) throws IOException {
     this.stateAccessor =
         new FnApiStateAccessor(
             pipelineOptions,
+            runnerCapabilities,
             pTransformId,
             processBundleInstructionId,
             cacheTokens,
@@ -2210,15 +2214,28 @@ public <T> void output(TupleTag<T> tag, T output) {
 
     @Override
     public void outputWithTimestamp(OutputT output, Instant timestamp) {
-      // TODO: Check that timestamp is valid once all runners can provide proper timestamps.
+      // TODO(https://github.com/apache/beam/issues/29637): Check that timestamp is valid once all
+      // runners can provide proper timestamps.
       outputTo(
           mainOutputConsumer,
           WindowedValue.of(output, timestamp, currentWindow, currentElement.getPane()));
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      // TODO(https://github.com/apache/beam/issues/29637): Check that timestamp is valid once all
+      // runners can provide proper timestamps.
+      outputTo(mainOutputConsumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+    }
+
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
-      // TODO: Check that timestamp is valid once all runners can provide proper timestamps.
+      // TODO(https://github.com/apache/beam/issues/29637): Check that timestamp is valid once all
+      // runners can provide proper timestamps.
       FnDataReceiver<WindowedValue<T>> consumer =
           (FnDataReceiver) localNameToConsumer.get(tag.getId());
       if (consumer == null) {
@@ -2228,6 +2245,23 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
           consumer, WindowedValue.of(output, timestamp, currentWindow, currentElement.getPane()));
     }
 
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      // TODO(https://github.com/apache/beam/issues/29637): Check that timestamp is valid once all
+      // runners can provide proper timestamps.
+      FnDataReceiver<WindowedValue<T>> consumer =
+          (FnDataReceiver) localNameToConsumer.get(tag.getId());
+      if (consumer == null) {
+        throw new IllegalArgumentException(String.format("Unknown output tag %s", tag));
+      }
+      outputTo(consumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+    }
+
     @Override
     public State state(String stateId, boolean alwaysFetched) {
       StateDeclaration stateDeclaration = doFnSignature.stateDeclarations().get(stateId);
@@ -2372,6 +2406,46 @@ public Instant timestamp(DoFn<InputT, OutputT> doFn) {
                   currentElement.getPane()));
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      checkTimestamp(timestamp);
+      double size =
+          doFnInvoker.invokeGetSize(
+              new DelegatingArgumentProvider<InputT, OutputT>(
+                  this, this.errorContextPrefix + "/GetSize") {
+                @Override
+                public Object restriction() {
+                  return output;
+                }
+
+                @Override
+                public Instant timestamp(DoFn<InputT, OutputT> doFn) {
+                  return timestamp;
+                }
+
+                @Override
+                public RestrictionTracker<?, ?> restrictionTracker() {
+                  return doFnInvoker.invokeNewTracker(this);
+                }
+              });
+
+      outputTo(
+          mainOutputConsumer,
+          (WindowedValue<OutputT>)
+              WindowedValue.of(
+                  KV.of(
+                      KV.of(
+                          currentElement.getValue(), KV.of(output, currentWatermarkEstimatorState)),
+                      size),
+                  timestamp,
+                  windows,
+                  paneInfo));
+    }
+
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       // Note that the OutputReceiver/RowOutputReceiver specifically will use the non-tag versions
@@ -2380,6 +2454,19 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
           String.format("Non-main output %s unsupported in %s", tag, errorContextPrefix));
     }
 
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      // Note that the OutputReceiver/RowOutputReceiver specifically will use the non-tag versions
+      // of these methods when producing output.
+      throw new UnsupportedOperationException(
+          String.format("Non-main output %s unsupported in %s", tag, errorContextPrefix));
+    }
+
     @Override
     public State state(String stateId, boolean alwaysFetched) {
       throw new UnsupportedOperationException(
@@ -2487,6 +2574,46 @@ public Instant timestamp(DoFn<InputT, OutputT> doFn) {
                   currentElement.getPane()));
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      checkTimestamp(timestamp);
+      double size =
+          doFnInvoker.invokeGetSize(
+              new DelegatingArgumentProvider<InputT, OutputT>(
+                  this, errorContextPrefix + "/GetSize") {
+                @Override
+                public Object restriction() {
+                  return output;
+                }
+
+                @Override
+                public Instant timestamp(DoFn<InputT, OutputT> doFn) {
+                  return timestamp;
+                }
+
+                @Override
+                public RestrictionTracker<?, ?> restrictionTracker() {
+                  return doFnInvoker.invokeNewTracker(this);
+                }
+              });
+
+      outputTo(
+          mainOutputConsumer,
+          (WindowedValue<OutputT>)
+              WindowedValue.of(
+                  KV.of(
+                      KV.of(
+                          currentElement.getValue(), KV.of(output, currentWatermarkEstimatorState)),
+                      size),
+                  timestamp,
+                  windows,
+                  paneInfo));
+    }
+
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       // Note that the OutputReceiver/RowOutputReceiver specifically will use the non-tag versions
@@ -2494,6 +2621,19 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
       throw new UnsupportedOperationException(
           String.format("Non-main output %s unsupported in %s", tag, errorContextPrefix));
     }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      // Note that the OutputReceiver/RowOutputReceiver specifically will use the non-tag versions
+      // of these methods when producing output.
+      throw new UnsupportedOperationException(
+          String.format("Non-main output %s unsupported in %s", tag, errorContextPrefix));
+    }
   }
 
   /** Provides arguments for a {@link DoFnInvoker} for a non-window observing method. */
@@ -2530,6 +2670,16 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
               output, timestamp, currentElement.getWindows(), currentElement.getPane()));
     }
 
+    @Override
+    public void outputWindowedValue(
+        OutputT output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      checkTimestamp(timestamp);
+      outputTo(mainOutputConsumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+    }
+
     @Override
     public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp) {
       checkTimestamp(timestamp);
@@ -2543,6 +2693,22 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
           WindowedValue.of(
               output, timestamp, currentElement.getWindows(), currentElement.getPane()));
     }
+
+    @Override
+    public <T> void outputWindowedValue(
+        TupleTag<T> tag,
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      checkTimestamp(timestamp);
+      FnDataReceiver<WindowedValue<T>> consumer =
+          (FnDataReceiver) localNameToConsumer.get(tag.getId());
+      if (consumer == null) {
+        throw new IllegalArgumentException(String.format("Unknown output tag %s", tag));
+      }
+      outputTo(consumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+    }
   }
 
   /** Provides base arguments for a {@link DoFnInvoker} for a non-window observing method. */
@@ -2672,6 +2838,16 @@ public void outputWithTimestamp(Row output, Instant timestamp) {
                 ProcessBundleContextBase.this.outputWithTimestamp(
                     fromRowFunction.apply(output), timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  Row output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                ProcessBundleContextBase.this.outputWindowedValue(
+                    fromRowFunction.apply(output), timestamp, windows, paneInfo);
+              }
             };
 
     @Override
@@ -2707,6 +2883,16 @@ public void output(T output) {
               public void outputWithTimestamp(T output, Instant timestamp) {
                 ProcessBundleContextBase.this.outputWithTimestamp(tag, output, timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  T output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                ProcessBundleContextBase.this.outputWindowedValue(
+                    tag, output, timestamp, windows, paneInfo);
+              }
             };
           }
 
@@ -2742,6 +2928,16 @@ public void outputWithTimestamp(Row output, Instant timestamp) {
                 ProcessBundleContextBase.this.outputWithTimestamp(
                     tag, fromRowFunction.apply(output), timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  Row output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                ProcessBundleContextBase.this.outputWindowedValue(
+                    tag, fromRowFunction.apply(output), timestamp, windows, paneInfo);
+              }
             };
           }
 
@@ -2856,6 +3052,16 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
             WindowedValue.of(output, timestamp, currentWindow, currentTimer.getPane()));
       }
 
+      @Override
+      public void outputWindowedValue(
+          OutputT output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        checkOnWindowExpirationTimestamp(timestamp);
+        outputTo(mainOutputConsumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+      }
+
       @Override
       public <T> void output(TupleTag<T> tag, T output) {
         FnDataReceiver<WindowedValue<T>> consumer =
@@ -2881,6 +3087,19 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
             consumer, WindowedValue.of(output, timestamp, currentWindow, currentTimer.getPane()));
       }
 
+      @Override
+      public <T> void outputWindowedValue(
+          TupleTag<T> tag,
+          T output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        checkOnWindowExpirationTimestamp(timestamp);
+        FnDataReceiver<WindowedValue<T>> consumer =
+            (FnDataReceiver) localNameToConsumer.get(tag.getId());
+        outputTo(consumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+      }
+
       @SuppressWarnings(
           "deprecation") // Allowed Skew is deprecated for users, but must be respected
       private void checkOnWindowExpirationTimestamp(Instant timestamp) {
@@ -2952,6 +3171,16 @@ public void output(Row output) {
               public void outputWithTimestamp(Row output, Instant timestamp) {
                 context.outputWithTimestamp(fromRowFunction.apply(output), timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  Row output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                context.outputWindowedValue(
+                    fromRowFunction.apply(output), timestamp, windows, paneInfo);
+              }
             };
 
     @Override
@@ -2984,6 +3213,15 @@ public void output(T output) {
               public void outputWithTimestamp(T output, Instant timestamp) {
                 context.outputWithTimestamp(tag, output, timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  T output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                context.outputWindowedValue(tag, output, timestamp, windows, paneInfo);
+              }
             };
           }
 
@@ -3015,6 +3253,16 @@ public void output(Row output) {
               public void outputWithTimestamp(Row output, Instant timestamp) {
                 context.outputWithTimestamp(tag, fromRowFunction.apply(output), timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  Row output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                context.outputWindowedValue(
+                    tag, fromRowFunction.apply(output), timestamp, windows, paneInfo);
+              }
             };
           }
 
@@ -3100,6 +3348,16 @@ public void outputWithTimestamp(OutputT output, Instant timestamp) {
             WindowedValue.of(output, timestamp, currentWindow, currentTimer.getPane()));
       }
 
+      @Override
+      public void outputWindowedValue(
+          OutputT output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        checkTimerTimestamp(timestamp);
+        outputTo(mainOutputConsumer, WindowedValue.of(output, timestamp, windows, paneInfo));
+      }
+
       @Override
       public <T> void output(TupleTag<T> tag, T output) {
         checkTimerTimestamp(currentTimer.getHoldTimestamp());
@@ -3126,6 +3384,14 @@ public <T> void outputWithTimestamp(TupleTag<T> tag, T output, Instant timestamp
             consumer, WindowedValue.of(output, timestamp, currentWindow, currentTimer.getPane()));
       }
 
+      @Override
+      public <T> void outputWindowedValue(
+          TupleTag<T> tag,
+          T output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {}
+
       @Override
       public TimeDomain timeDomain() {
         return currentTimeDomain;
@@ -3212,6 +3478,16 @@ public void output(Row output) {
               public void outputWithTimestamp(Row output, Instant timestamp) {
                 context.outputWithTimestamp(fromRowFunction.apply(output), timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  Row output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                context.outputWindowedValue(
+                    fromRowFunction.apply(output), timestamp, windows, paneInfo);
+              }
             };
 
     @Override
@@ -3244,6 +3520,15 @@ public void output(T output) {
               public void outputWithTimestamp(T output, Instant timestamp) {
                 context.outputWithTimestamp(tag, output, timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  T output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                context.outputWindowedValue(tag, output, timestamp, windows, paneInfo);
+              }
             };
           }
 
@@ -3275,6 +3560,16 @@ public void output(Row output) {
               public void outputWithTimestamp(Row output, Instant timestamp) {
                 context.outputWithTimestamp(tag, fromRowFunction.apply(output), timestamp);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  Row output,
+                  Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                context.outputWindowedValue(
+                    tag, fromRowFunction.apply(output), timestamp, windows, paneInfo);
+              }
             };
           }
 
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
index 69d9a1ff6c8e..204c491dc102 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/FnApiStateAccessor.java
@@ -26,6 +26,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Supplier;
 import org.apache.beam.fn.harness.Cache;
@@ -33,7 +34,9 @@
 import org.apache.beam.model.fnexecution.v1.BeamFnApi;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest.CacheToken;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
 import org.apache.beam.runners.core.SideInputReader;
+import org.apache.beam.runners.core.construction.BeamUrns;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.VoidCoder;
@@ -74,6 +77,7 @@
 })
 public class FnApiStateAccessor<K> implements SideInputReader, StateBinder {
   private final PipelineOptions pipelineOptions;
+  private final Set<String> runnerCapabilites;
   private final Map<StateKey, Object> stateKeyObjectCache;
   private final Map<TupleTag<?>, SideInputSpec> sideInputSpecMap;
   private final BeamFnStateClient beamFnStateClient;
@@ -91,6 +95,7 @@ public class FnApiStateAccessor<K> implements SideInputReader, StateBinder {
 
   public FnApiStateAccessor(
       PipelineOptions pipelineOptions,
+      Set<String> runnerCapabilites,
       String ptransformId,
       Supplier<String> processBundleInstructionId,
       Supplier<List<CacheToken>> cacheTokens,
@@ -103,6 +108,7 @@ public FnApiStateAccessor(
       Supplier<K> currentKeySupplier,
       Supplier<BoundedWindow> currentWindowSupplier) {
     this.pipelineOptions = pipelineOptions;
+    this.runnerCapabilites = runnerCapabilites;
     this.stateKeyObjectCache = Maps.newHashMap();
     this.sideInputSpecMap = sideInputSpecMap;
     this.beamFnStateClient = beamFnStateClient;
@@ -238,7 +244,11 @@ public ResultT get() {
                               processBundleInstructionId.get(),
                               key,
                               ((KvCoder) sideInputSpec.getCoder()).getKeyCoder(),
-                              ((KvCoder) sideInputSpec.getCoder()).getValueCoder()));
+                              ((KvCoder) sideInputSpec.getCoder()).getValueCoder(),
+                              runnerCapabilites.contains(
+                                  BeamUrns.getUrn(
+                                      RunnerApi.StandardRunnerProtocols.Enum
+                                          .MULTIMAP_KEYS_VALUES_SIDE_INPUT))));
                 default:
                   throw new IllegalStateException(
                       String.format(
diff --git a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java
index 4bba03875774..0c7726441021 100644
--- a/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java
+++ b/sdks/java/harness/src/main/java/org/apache/beam/fn/harness/state/MultimapSideInput.java
@@ -20,13 +20,21 @@
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.Function;
 import org.apache.beam.fn.harness.Cache;
 import org.apache.beam.fn.harness.Caches;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.transforms.Materializations.MultimapView;
 import org.apache.beam.sdk.util.ByteStringOutputStream;
+import org.apache.beam.sdk.values.KV;
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString;
 
 /**
@@ -38,11 +46,15 @@
 })
 public class MultimapSideInput<K, V> implements MultimapView<K, V> {
 
+  private static final int BULK_READ_SIZE = 100;
+
   private final Cache<?, ?> cache;
   private final BeamFnStateClient beamFnStateClient;
   private final StateRequest keysRequest;
   private final Coder<K> keyCoder;
   private final Coder<V> valueCoder;
+  private volatile Function<ByteString, Iterable<V>> bulkReadResult;
+  private final boolean useBulkRead;
 
   public MultimapSideInput(
       Cache<?, ?> cache,
@@ -50,7 +62,8 @@ public MultimapSideInput(
       String instructionId,
       StateKey stateKey,
       Coder<K> keyCoder,
-      Coder<V> valueCoder) {
+      Coder<V> valueCoder,
+      boolean useBulkRead) {
     checkArgument(
         stateKey.hasMultimapKeysSideInput(),
         "Expected MultimapKeysSideInput StateKey but received %s.",
@@ -61,6 +74,7 @@ public MultimapSideInput(
         StateRequest.newBuilder().setInstructionId(instructionId).setStateKey(stateKey).build();
     this.keyCoder = keyCoder;
     this.valueCoder = valueCoder;
+    this.useBulkRead = useBulkRead;
   }
 
   @Override
@@ -71,17 +85,74 @@ public Iterable<K> get() {
 
   @Override
   public Iterable<V> get(K k) {
-    ByteStringOutputStream output = new ByteStringOutputStream();
-    try {
-      keyCoder.encode(k, output);
-    } catch (IOException e) {
-      throw new IllegalStateException(
-          String.format(
-              "Failed to encode key %s for side input id %s.",
-              k, keysRequest.getStateKey().getMultimapKeysSideInput().getSideInputId()),
-          e);
+    ByteString encodedKey = encodeKey(k);
+
+    if (useBulkRead) {
+      if (bulkReadResult == null) {
+        synchronized (this) {
+          if (bulkReadResult == null) {
+            Map<ByteString, Iterable<V>> bulkRead = new HashMap<>();
+            StateKey bulkReadStateKey =
+                StateKey.newBuilder()
+                    .setMultimapKeysValuesSideInput(
+                        StateKey.MultimapKeysValuesSideInput.newBuilder()
+                            .setTransformId(
+                                keysRequest
+                                    .getStateKey()
+                                    .getMultimapKeysSideInput()
+                                    .getTransformId())
+                            .setSideInputId(
+                                keysRequest
+                                    .getStateKey()
+                                    .getMultimapKeysSideInput()
+                                    .getSideInputId())
+                            .setWindow(
+                                keysRequest.getStateKey().getMultimapKeysSideInput().getWindow()))
+                    .build();
+
+            StateRequest bulkReadRequest =
+                keysRequest.toBuilder().setStateKey(bulkReadStateKey).build();
+            try {
+              Iterator<KV<K, Iterable<V>>> entries =
+                  StateFetchingIterators.readAllAndDecodeStartingFrom(
+                          Caches.subCache(cache, "ValuesForKey", encodedKey),
+                          beamFnStateClient,
+                          bulkReadRequest,
+                          KvCoder.of(keyCoder, IterableCoder.of(valueCoder)))
+                      .iterator();
+              while (bulkRead.size() < BULK_READ_SIZE && entries.hasNext()) {
+                KV<K, Iterable<V>> entry = entries.next();
+                bulkRead.put(encodeKey(entry.getKey()), entry.getValue());
+              }
+              if (entries.hasNext()) {
+                bulkReadResult = bulkRead::get;
+              } else {
+                bulkReadResult =
+                    key -> {
+                      Iterable<V> result = bulkRead.get(key);
+                      if (result == null) {
+                        // As we read the entire set of values, we don't have to do a lookup to know
+                        // this key doesn't exist.
+                        // Missing keys are treated as empty iterables in this multimap.
+                        return Collections.emptyList();
+                      } else {
+                        return result;
+                      }
+                    };
+              }
+            } catch (Exception exn) {
+              bulkReadResult = bulkRead::get;
+            }
+          }
+        }
+      }
+
+      Iterable<V> bulkReadValues = bulkReadResult.apply(encodedKey);
+      if (bulkReadValues != null) {
+        return bulkReadValues;
+      }
     }
-    ByteString encodedKey = output.toByteString();
+
     StateKey stateKey =
         StateKey.newBuilder()
             .setMultimapSideInput(
@@ -98,4 +169,18 @@ public Iterable<V> get(K k) {
     return StateFetchingIterators.readAllAndDecodeStartingFrom(
         Caches.subCache(cache, "ValuesForKey", encodedKey), beamFnStateClient, request, valueCoder);
   }
+
+  private ByteString encodeKey(K k) {
+    ByteStringOutputStream output = new ByteStringOutputStream();
+    try {
+      keyCoder.encode(k, output);
+    } catch (IOException e) {
+      throw new IllegalStateException(
+          String.format(
+              "Failed to encode key %s for side input id %s.",
+              k, keysRequest.getStateKey().getMultimapKeysSideInput().getSideInputId()),
+          e);
+    }
+    return output.toByteString();
+  }
 }
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
index 4aaaa3d945d9..5447d6f0c6d6 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/FakeBeamFnStateClient.java
@@ -136,6 +136,10 @@ public CompletableFuture<StateResponse> handle(StateRequest.Builder requestBuild
     if (key.getTypeCase() == TypeCase.MULTIMAP_SIDE_INPUT || key.getTypeCase() == TypeCase.RUNNER) {
       assertEquals(RequestCase.GET, request.getRequestCase());
     }
+    if (key.getTypeCase() == TypeCase.MULTIMAP_KEYS_VALUES_SIDE_INPUT && !data.containsKey(key)) {
+      // Allow testing this not being supported rather than blindly returning the empty list.
+      throw new UnsupportedOperationException("No multimap keys values states provided.");
+    }
 
     switch (request.getRequestCase()) {
       case GET:
diff --git a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java
index a4c20e3593ac..23a572894b40 100644
--- a/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java
+++ b/sdks/java/harness/src/test/java/org/apache/beam/fn/harness/state/MultimapSideInputTest.java
@@ -27,6 +27,8 @@
 import org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey;
 import org.apache.beam.sdk.coders.ByteArrayCoder;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.IterableCoder;
+import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.util.ByteStringOutputStream;
 import org.apache.beam.sdk.values.KV;
@@ -50,12 +52,39 @@ public class MultimapSideInputTest {
   private static final byte[] B = "B".getBytes(StandardCharsets.UTF_8);
   private static final byte[] UNKNOWN = "UNKNOWN".getBytes(StandardCharsets.UTF_8);
 
+  @Test
+  public void testGetWithBulkRead() throws Exception {
+    FakeBeamFnStateClient fakeBeamFnStateClient =
+        new FakeBeamFnStateClient(
+            ImmutableMap.of(
+                keysValuesStateKey(),
+                KV.of(
+                    KvCoder.of(ByteArrayCoder.of(), IterableCoder.of(StringUtf8Coder.of())),
+                    asList(KV.of(A, asList("A1", "A2", "A3")), KV.of(B, asList("B1", "B2"))))));
+
+    MultimapSideInput<byte[], String> multimapSideInput =
+        new MultimapSideInput<>(
+            Caches.noop(),
+            fakeBeamFnStateClient,
+            "instructionId",
+            keysStateKey(),
+            ByteArrayCoder.of(),
+            StringUtf8Coder.of(),
+            true);
+    assertArrayEquals(
+        new String[] {"A1", "A2", "A3"}, Iterables.toArray(multimapSideInput.get(A), String.class));
+    assertArrayEquals(
+        new String[] {"B1", "B2"}, Iterables.toArray(multimapSideInput.get(B), String.class));
+    assertArrayEquals(
+        new String[] {}, Iterables.toArray(multimapSideInput.get(UNKNOWN), String.class));
+  }
+
   @Test
   public void testGet() throws Exception {
     FakeBeamFnStateClient fakeBeamFnStateClient =
         new FakeBeamFnStateClient(
             ImmutableMap.of(
-                stateKey(), KV.of(ByteArrayCoder.of(), asList(A, B)),
+                keysStateKey(), KV.of(ByteArrayCoder.of(), asList(A, B)),
                 key(A), KV.of(StringUtf8Coder.of(), asList("A1", "A2", "A3")),
                 key(B), KV.of(StringUtf8Coder.of(), asList("B1", "B2"))));
 
@@ -64,9 +93,10 @@ public void testGet() throws Exception {
             Caches.noop(),
             fakeBeamFnStateClient,
             "instructionId",
-            stateKey(),
+            keysStateKey(),
             ByteArrayCoder.of(),
-            StringUtf8Coder.of());
+            StringUtf8Coder.of(),
+            true);
     assertArrayEquals(
         new String[] {"A1", "A2", "A3"}, Iterables.toArray(multimapSideInput.get(A), String.class));
     assertArrayEquals(
@@ -82,7 +112,7 @@ public void testGetCached() throws Exception {
     FakeBeamFnStateClient fakeBeamFnStateClient =
         new FakeBeamFnStateClient(
             ImmutableMap.of(
-                stateKey(), KV.of(ByteArrayCoder.of(), asList(A, B)),
+                keysStateKey(), KV.of(ByteArrayCoder.of(), asList(A, B)),
                 key(A), KV.of(StringUtf8Coder.of(), asList("A1", "A2", "A3")),
                 key(B), KV.of(StringUtf8Coder.of(), asList("B1", "B2"))));
 
@@ -94,9 +124,10 @@ public void testGetCached() throws Exception {
               cache,
               fakeBeamFnStateClient,
               "instructionId",
-              stateKey(),
+              keysStateKey(),
               ByteArrayCoder.of(),
-              StringUtf8Coder.of());
+              StringUtf8Coder.of(),
+              true);
       assertArrayEquals(
           new String[] {"A1", "A2", "A3"},
           Iterables.toArray(multimapSideInput.get(A), String.class));
@@ -117,9 +148,10 @@ public void testGetCached() throws Exception {
                 throw new IllegalStateException("Unexpected call for test.");
               },
               "instructionId",
-              stateKey(),
+              keysStateKey(),
               ByteArrayCoder.of(),
-              StringUtf8Coder.of());
+              StringUtf8Coder.of(),
+              true);
       assertArrayEquals(
           new String[] {"A1", "A2", "A3"},
           Iterables.toArray(multimapSideInput.get(A), String.class));
@@ -132,7 +164,7 @@ public void testGetCached() throws Exception {
     }
   }
 
-  private StateKey stateKey() throws IOException {
+  private StateKey keysStateKey() throws IOException {
     return StateKey.newBuilder()
         .setMultimapKeysSideInput(
             StateKey.MultimapKeysSideInput.newBuilder()
@@ -142,6 +174,16 @@ private StateKey stateKey() throws IOException {
         .build();
   }
 
+  private StateKey keysValuesStateKey() throws IOException {
+    return StateKey.newBuilder()
+        .setMultimapKeysValuesSideInput(
+            StateKey.MultimapKeysValuesSideInput.newBuilder()
+                .setTransformId("ptransformId")
+                .setSideInputId("sideInputId")
+                .setWindow(ByteString.copyFromUtf8("encodedWindow")))
+        .build();
+  }
+
   private StateKey key(byte[] key) throws IOException {
     ByteStringOutputStream out = new ByteStringOutputStream();
     ByteArrayCoder.of().encode(key, out);
diff --git a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java
index 52d520a0a191..7ef643488a23 100644
--- a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java
+++ b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIO.java
@@ -27,6 +27,7 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 import java.util.stream.Collectors;
@@ -109,6 +110,7 @@
  * <tr><td>{@link TableSchema.TypeName#ENUM8}</td> <td>{@link Schema.TypeName#STRING}</td></tr>
  * <tr><td>{@link TableSchema.TypeName#ENUM16}</td> <td>{@link Schema.TypeName#STRING}</td></tr>
  * <tr><td>{@link TableSchema.TypeName#BOOL}</td> <td>{@link Schema.TypeName#BOOLEAN}</td></tr>
+ * <tr><td>{@link TableSchema.TypeName#TUPLE}</td> <td>{@link Schema.TypeName#ROW}</td></tr>
  * </table>
  *
  * Nullable row columns are supported through Nullable type in ClickHouse. Low cardinality hint is
@@ -475,6 +477,15 @@ abstract static class Builder<T> {
     }
   }
 
+  private static String tuplePreprocessing(String payload) {
+    List<String> l =
+        Arrays.stream(payload.trim().split(","))
+            .map(s -> s.trim().replaceAll(" +", "' "))
+            .collect(Collectors.toList());
+    String content =
+        String.join(",", l).trim().replaceAll("Tuple\\(", "Tuple('").replaceAll(",", ",'");
+    return content;
+  }
   /**
    * Returns {@link TableSchema} for a given table.
    *
@@ -498,7 +509,13 @@ public static TableSchema getTableSchema(String jdbcUrl, String table) {
           String defaultTypeStr = rs.getString("default_type");
           String defaultExpression = rs.getString("default_expression");
 
-          ColumnType columnType = ColumnType.parse(type);
+          ColumnType columnType = null;
+          if (type.toLowerCase().trim().startsWith("tuple(")) {
+            String content = tuplePreprocessing(type);
+            columnType = ColumnType.parse(content);
+          } else {
+            columnType = ColumnType.parse(type);
+          }
           DefaultType defaultType = DefaultType.parse(defaultTypeStr).orElse(null);
 
           Object defaultValue;
diff --git a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java
index 8ed62eee3b59..c8c49a656e3b 100644
--- a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java
+++ b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/ClickHouseWriter.java
@@ -21,9 +21,11 @@
 import com.clickhouse.client.ClickHousePipedOutputStream;
 import com.clickhouse.client.data.BinaryStreamUtils;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 import org.apache.beam.sdk.io.clickhouse.TableSchema.ColumnType;
 import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.RowWithStorage;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions;
 import org.joda.time.Days;
@@ -146,6 +148,20 @@ static void writeValue(ClickHouseOutputStream stream, ColumnType columnType, Obj
       case BOOL:
         BinaryStreamUtils.writeBoolean(stream, (Boolean) value);
         break;
+      case TUPLE:
+        RowWithStorage rowValues = (RowWithStorage) value;
+        List<Object> tupleValues = rowValues.getValues();
+        Collection<ColumnType> columnTypesList = columnType.tupleTypes().values();
+        int index = 0;
+        for (ColumnType ct : columnTypesList) {
+          if (ct.nullable()) {
+            writeNullableValue(stream, ct, tupleValues.get(index));
+          } else {
+            writeValue(stream, ct, tupleValues.get(index));
+          }
+          index++;
+        }
+        break;
     }
   }
 
diff --git a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/TableSchema.java b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/TableSchema.java
index 06ba2399a3cd..b89a88b3fae8 100644
--- a/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/TableSchema.java
+++ b/sdks/java/io/clickhouse/src/main/java/org/apache/beam/sdk/io/clickhouse/TableSchema.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import org.apache.beam.sdk.schemas.Schema;
 import org.apache.beam.sdk.schemas.logicaltypes.FixedBytes;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -111,6 +112,14 @@ public static Schema.FieldType getEquivalentFieldType(ColumnType columnType) {
         return Schema.FieldType.STRING;
       case BOOL:
         return Schema.FieldType.BOOLEAN;
+      case TUPLE:
+        List<Schema.Field> fields =
+            columnType.tupleTypes().entrySet().stream()
+                .map(x -> Schema.Field.of(x.getKey(), Schema.FieldType.DATETIME))
+                .collect(Collectors.toList());
+        Schema.Field[] array = fields.toArray(new Schema.Field[fields.size()]);
+        Schema schema = Schema.of(array);
+        return Schema.FieldType.row(schema);
     }
 
     // not possible, errorprone checks for exhaustive switch
@@ -168,7 +177,9 @@ public enum TypeName {
     // Composite type
     ARRAY,
     // Primitive type
-    BOOL
+    BOOL,
+    // Composite type
+    TUPLE
   }
 
   /**
@@ -208,6 +219,7 @@ public abstract static class ColumnType implements Serializable {
     public static final ColumnType UINT32 = ColumnType.of(TypeName.UINT32);
     public static final ColumnType UINT64 = ColumnType.of(TypeName.UINT64);
     public static final ColumnType BOOL = ColumnType.of(TypeName.BOOL);
+    public static final ColumnType TUPLE = ColumnType.of(TypeName.TUPLE);
 
     // ClickHouse doesn't allow nested nullables, so boolean flag is enough
     public abstract boolean nullable();
@@ -220,6 +232,8 @@ public abstract static class ColumnType implements Serializable {
 
     public abstract @Nullable ColumnType arrayElementType();
 
+    public abstract @Nullable Map<String, ColumnType> tupleTypes();
+
     public ColumnType withNullable(boolean nullable) {
       return toBuilder().nullable(nullable).build();
     }
@@ -265,6 +279,14 @@ public static ColumnType array(ColumnType arrayElementType) {
           .build();
     }
 
+    public static ColumnType tuple(Map<String, ColumnType> elements) {
+      return ColumnType.builder()
+          .typeName(TypeName.TUPLE)
+          .nullable(false)
+          .tupleTypes(elements)
+          .build();
+    }
+
     /**
      * Parse string with ClickHouse type to {@link ColumnType}.
      *
@@ -339,6 +361,8 @@ abstract static class Builder {
 
       public abstract Builder fixedStringSize(Integer size);
 
+      public abstract Builder tupleTypes(Map<String, ColumnType> tupleElements);
+
       public abstract ColumnType build();
     }
   }
diff --git a/sdks/java/io/clickhouse/src/main/javacc/ColumnTypeParser.jj b/sdks/java/io/clickhouse/src/main/javacc/ColumnTypeParser.jj
index abe29aff3f8d..5bb9ba4171a6 100644
--- a/sdks/java/io/clickhouse/src/main/javacc/ColumnTypeParser.jj
+++ b/sdks/java/io/clickhouse/src/main/javacc/ColumnTypeParser.jj
@@ -17,6 +17,10 @@
  */
 options {
     IGNORE_CASE=true;
+    DEBUG_PARSER = false;
+    DEBUG_LOOKAHEAD = false;
+    DEBUG_TOKEN_MANAGER = false;
+    STATIC = false;
 }
 
 PARSER_BEGIN(ColumnTypeParser)
@@ -99,6 +103,7 @@ TOKEN :
   | < EQ             : "=" >
   | < BOOL           : "BOOL" >
   | < LOWCARDINALITY : "LOWCARDINALITY" >
+  | < TUPLE          : "TUPLE" >
 }
 
 public ColumnType columnType() :
@@ -113,6 +118,7 @@ public ColumnType columnType() :
         | ct = array()
         | ct = nullable()
         | ct = lowcardenality()
+        | ct = tuple()
     )
     {
         return ct;
@@ -263,6 +269,33 @@ private Map<String, Integer> enumElements() :
     }
 }
 
+private Map.Entry<String, ColumnType> tupleElement() :
+{
+    String key;
+    ColumnType value;
+    Token token;
+}
+{
+    ( (key = string() ) ( value = columnType() ) )  {
+        return Maps.immutableEntry(key, value);
+    }
+}
+
+private Map<String, ColumnType> tupleElements() :
+{
+    Map.Entry<String, ColumnType> el;
+    List<Map.Entry<String, ColumnType>> entries = Lists.newArrayList();
+}
+{
+    (
+                  ( el = tupleElement() { entries.add(el); } )
+        ( <COMMA> ( el = tupleElement() { entries.add(el); } ) )*
+    )
+    {
+        return ImmutableMap.copyOf(entries);
+    }
+}
+
 private ColumnType enum_() :
 {
     Map<String, Integer> elements;
@@ -289,4 +322,18 @@ private ColumnType lowcardenality() :
     (
         (<LOWCARDINALITY> <LPAREN> (ct = primitive()) <RPAREN>) { return ct; }
     )
+}
+
+private ColumnType tuple() :
+{
+    Map<String, ColumnType> elements;
+}
+{
+    (
+        (<TUPLE> <LPAREN> ( elements = tupleElements() ) <RPAREN>)
+        {
+            return ColumnType.tuple(elements);
+        }
+    )
+
 }
\ No newline at end of file
diff --git a/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOTest.java b/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOTest.java
index 33fe9467d45b..b31a19236cb0 100644
--- a/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOTest.java
+++ b/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/ClickHouseIOTest.java
@@ -138,6 +138,84 @@ public void testArrayOfArrayOfInt64() throws Exception {
     assertEquals(15L, sum0);
   }
 
+  @Test
+  public void testTupleType() throws Exception {
+    Schema tupleSchema =
+        Schema.of(
+            Schema.Field.of("f0", FieldType.STRING), Schema.Field.of("f1", FieldType.BOOLEAN));
+    Schema schema = Schema.of(Schema.Field.of("t0", FieldType.row(tupleSchema)));
+    Row row1Tuple = Row.withSchema(tupleSchema).addValue("tuple").addValue(true).build();
+
+    Row row1 = Row.withSchema(schema).addValue(row1Tuple).build();
+
+    executeSql(
+        "CREATE TABLE test_named_tuples (" + "t0 Tuple(`f0` String, `f1` Bool)" + ") ENGINE=Log");
+
+    pipeline.apply(Create.of(row1).withRowSchema(schema)).apply(write("test_named_tuples"));
+
+    pipeline.run().waitUntilFinish();
+
+    try (ResultSet rs = executeQuery("SELECT * FROM test_named_tuples")) {
+      rs.next();
+      assertEquals("('tuple',true)", rs.getString("t0"));
+    }
+
+    try (ResultSet rs = executeQuery("SELECT t0.f0 as f0, t0.f1 as f1 FROM test_named_tuples")) {
+      rs.next();
+      assertEquals("tuple", rs.getString("f0"));
+      assertEquals("true", rs.getString("f1"));
+    }
+  }
+
+  @Test
+  public void testComplexTupleType() throws Exception {
+    Schema sizeSchema =
+        Schema.of(
+            Schema.Field.of("width", FieldType.INT64.withNullable(true)),
+            Schema.Field.of("height", FieldType.INT64.withNullable(true)));
+
+    Schema browserSchema =
+        Schema.of(
+            Schema.Field.of("name", FieldType.STRING.withNullable(true)),
+            Schema.Field.of("size", FieldType.row(sizeSchema)),
+            Schema.Field.of("version", FieldType.STRING.withNullable(true)));
+
+    Schema propSchema =
+        Schema.of(
+            Schema.Field.of("browser", FieldType.row(browserSchema)),
+            Schema.Field.of("deviceCategory", FieldType.STRING.withNullable(true)));
+
+    Schema schema = Schema.of(Schema.Field.of("prop", FieldType.row(propSchema)));
+
+    Row sizeRow = Row.withSchema(sizeSchema).addValue(10L).addValue(20L).build();
+    Row browserRow =
+        Row.withSchema(browserSchema).addValue("test").addValue(sizeRow).addValue("1.0.0").build();
+    Row propRow = Row.withSchema(propSchema).addValue(browserRow).addValue("mobile").build();
+    Row row1 = Row.withSchema(schema).addValue(propRow).build();
+
+    executeSql(
+        "CREATE TABLE test_named_complex_tuples ("
+            + "`prop` Tuple(`browser` Tuple(`name` Nullable(String),`size` Tuple(`width` Nullable(Int64), `height` Nullable(Int64)),`version` Nullable(String)),`deviceCategory` Nullable(String))"
+            + ") ENGINE=Log");
+
+    pipeline.apply(Create.of(row1).withRowSchema(schema)).apply(write("test_named_complex_tuples"));
+
+    pipeline.run().waitUntilFinish();
+
+    try (ResultSet rs = executeQuery("SELECT * FROM test_named_complex_tuples")) {
+      rs.next();
+      assertEquals("(('test',(10,20),'1.0.0'),'mobile')", rs.getString("prop"));
+    }
+
+    try (ResultSet rs =
+        executeQuery(
+            "SELECT prop.browser.name as name, prop.browser.size as size FROM test_named_complex_tuples")) {
+      rs.next();
+      assertEquals("test", rs.getString("name"));
+      assertEquals("(10,20)", rs.getString("size"));
+    }
+  }
+
   @Test
   public void testPrimitiveTypes() throws Exception {
     Schema schema =
diff --git a/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java b/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java
index 174761403471..f560d6268afb 100644
--- a/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java
+++ b/sdks/java/io/clickhouse/src/test/java/org/apache/beam/sdk/io/clickhouse/TableSchemaTest.java
@@ -19,6 +19,7 @@
 
 import static org.junit.Assert.assertEquals;
 
+import java.util.HashMap;
 import java.util.Map;
 import org.apache.beam.sdk.io.clickhouse.TableSchema.ColumnType;
 import org.apache.beam.sdk.schemas.Schema;
@@ -196,4 +197,54 @@ public void testEquivalentSchema() {
 
     assertEquals(expected, TableSchema.getEquivalentSchema(tableSchema));
   }
+
+  @Test
+  public void testParseTupleSingle() {
+    Map<String, ColumnType> m1 = new HashMap<>();
+    m1.put("s", ColumnType.STRING);
+    ColumnType columnType01 = ColumnType.parse("Tuple('s' String)");
+    assertEquals(ColumnType.tuple(m1), columnType01);
+  }
+
+  @Test
+  public void testParseTupleDouble() {
+    Map<String, ColumnType> m2 = new HashMap<>();
+    m2.put("a1", ColumnType.STRING);
+    m2.put("b", ColumnType.BOOL);
+    ColumnType columnType02 = ColumnType.parse("Tuple('a1' String,'b' Bool)");
+    assertEquals(ColumnType.tuple(m2), columnType02);
+  }
+
+  @Test
+  public void testTupleNested() {
+    Map<String, ColumnType> m1 = new HashMap<>();
+    m1.put("a", ColumnType.STRING);
+    Map<String, ColumnType> m3 = new HashMap<>();
+    m3.put("a", ColumnType.STRING);
+    m3.put("b", ColumnType.BOOL);
+    m3.put("c", ColumnType.tuple(m1));
+    ColumnType columnType03 = ColumnType.parse("Tuple('a' String,'b' Bool,'c' Tuple('a' String))");
+    assertEquals(ColumnType.tuple(m3), columnType03);
+  }
+
+  @Test
+  public void testTupleComplex() {
+    Map<String, ColumnType> m1 = new HashMap<>();
+    m1.put("width", ColumnType.INT64.withNullable(true));
+    m1.put("height", ColumnType.INT64.withNullable(true));
+
+    Map<String, ColumnType> m2 = new HashMap<>();
+    m2.put("name", ColumnType.STRING.withNullable(true));
+    m2.put("size", ColumnType.tuple(m1));
+    m2.put("version", ColumnType.STRING.withNullable(true));
+
+    Map<String, ColumnType> m3 = new HashMap<>();
+    m3.put("browser", ColumnType.tuple(m2));
+    m3.put("deviceCategory", ColumnType.STRING.withNullable(true));
+
+    ColumnType columnType03 =
+        ColumnType.parse(
+            "Tuple('browser' Tuple('name' Nullable(String),'size' Tuple('width' Nullable(Int64),'height' Nullable(Int64)),'version' Nullable(String)),'deviceCategory' Nullable(String))");
+    assertEquals(ColumnType.tuple(m3), columnType03);
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
index ee21e1185f37..3094af5855e9 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendClientInfo.java
@@ -105,7 +105,7 @@ public AppendClientInfo withNoAppendClient() {
   }
 
   public AppendClientInfo withAppendClient(
-      BigQueryServices.DatasetService datasetService,
+      BigQueryServices.WriteStreamService writeStreamService,
       Supplier<String> getStreamName,
       boolean useConnectionPool,
       AppendRowsRequest.MissingValueInterpretation missingValueInterpretation)
@@ -117,7 +117,7 @@ public AppendClientInfo withAppendClient(
       return toBuilder()
           .setStreamName(streamName)
           .setStreamAppendClient(
-              datasetService.getStreamAppendClient(
+              writeStreamService.getStreamAppendClient(
                   streamName, getDescriptor(), useConnectionPool, missingValueInterpretation))
           .build();
     }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 86f7a7f122b7..4424b53f83ce 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -577,7 +577,7 @@ public class BigQueryIO {
   private static final String DATASET_REGEXP = "[-\\w.]{1,1024}";
 
   /** Regular expression that matches Table IDs. */
-  private static final String TABLE_REGEXP = "[-\\w$@]{1,1024}";
+  private static final String TABLE_REGEXP = "[-\\w$@ ]{1,1024}";
 
   /**
    * Matches table specifications in the form {@code "[project_id]:[dataset_id].[table_id]"} or
@@ -2101,7 +2101,9 @@ static List<ResourceId> getExtractFilePaths(String extractDestinationDir, Job ex
    * <p>By default, tables will be created if they do not exist, which corresponds to a {@link
    * Write.CreateDisposition#CREATE_IF_NEEDED} disposition that matches the default of BigQuery's
    * Jobs API. A schema must be provided (via {@link Write#withSchema(TableSchema)}), or else the
-   * transform may fail at runtime with an {@link IllegalArgumentException}.
+   * transform may fail at runtime with an {@link IllegalArgumentException}. When updating a
+   * pipeline with a new schema, the existing schmea fields must stay in the same order, or the
+   * pipeline will break.
    *
    * <p>By default, writes require an empty table, which corresponds to a {@link
    * Write.WriteDisposition#WRITE_EMPTY} disposition that matches the default of BigQuery's Jobs
@@ -2539,7 +2541,9 @@ public enum WriteDisposition {
      *
      * <p>Note from the BigQuery API doc -- Schema update options are supported in two cases: when
      * writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination
-     * table is a partition of a table, specified by partition decorators.
+     * table is a partition of a table, specified by partition decorators. When updating a pipeline
+     * with a new schema, the existing schmea fields must stay in the same order, or the pipeline
+     * will break.
      *
      * @see <a
      *     href="https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#jobconfigurationquery">
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTranslation.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTranslation.java
new file mode 100644
index 000000000000..a3a270a315b2
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTranslation.java
@@ -0,0 +1,827 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static org.apache.beam.runners.core.construction.TransformUpgrader.fromByteArray;
+import static org.apache.beam.runners.core.construction.TransformUpgrader.toByteArray;
+
+import com.google.api.services.bigquery.model.Clustering;
+import com.google.api.services.bigquery.model.TableRow;
+import com.google.auto.service.AutoService;
+import com.google.cloud.bigquery.storage.v1.AppendRowsRequest.MissingValueInterpretation;
+import com.google.cloud.bigquery.storage.v1.DataFormat;
+import java.io.IOException;
+import java.io.InvalidClassException;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.beam.model.pipeline.v1.RunnerApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec;
+import org.apache.beam.runners.core.construction.PTransformTranslation.TransformPayloadTranslator;
+import org.apache.beam.runners.core.construction.SdkComponents;
+import org.apache.beam.runners.core.construction.TransformPayloadTranslatorRegistrar;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.FromBeamRowFunction;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.QueryPriority;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.ToBeamRowFunction;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.SchemaUpdateOption;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.RowWriterFactory.AvroRowWriterFactory;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
+import org.apache.beam.sdk.runners.AppliedPTransform;
+import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.Schema.FieldType;
+import org.apache.beam.sdk.schemas.logicaltypes.NanosDuration;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.ValueInSingleWindow;
+import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings({"rawtypes", "nullness"})
+public class BigQueryIOTranslation {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BigQueryIOTranslation.class);
+
+  static class BigQueryIOReadTranslator implements TransformPayloadTranslator<TypedRead<?>> {
+
+    static Schema schema =
+        Schema.builder()
+            .addNullableStringField("json_table_ref")
+            .addNullableStringField("query")
+            .addNullableBooleanField("validate")
+            .addNullableBooleanField("flatten_results")
+            .addNullableBooleanField("use_legacy_sql")
+            .addNullableBooleanField("with_template_compatibility")
+            .addNullableByteArrayField("bigquery_services")
+            .addNullableByteArrayField("parse_fn")
+            .addNullableByteArrayField("datum_reader_factory")
+            .addNullableByteArrayField("query_priority")
+            .addNullableStringField("query_location")
+            .addNullableStringField("query_temp_dataset")
+            .addNullableByteArrayField("method")
+            .addNullableByteArrayField("format")
+            .addNullableArrayField("selected_fields", FieldType.STRING)
+            .addNullableStringField("row_restriction")
+            .addNullableByteArrayField("coder")
+            .addNullableStringField("kms_key")
+            .addNullableByteArrayField("type_descriptor")
+            .addNullableByteArrayField("to_beam_row_fn")
+            .addNullableStringField("from_beam_row_fn")
+            .addNullableBooleanField("use_avro_logical_types")
+            .addNullableBooleanField("projection_pushdown_applied")
+            .build();
+
+    public static final String BIGQUERY_READ_TRANSFORM_URN =
+        "beam:transform:org.apache.beam:bigquery_read:v1";
+
+    @Override
+    public String getUrn() {
+      return BIGQUERY_READ_TRANSFORM_URN;
+    }
+
+    @Override
+    public RunnerApi.@Nullable FunctionSpec translate(
+        AppliedPTransform<?, ?, TypedRead<?>> application, SdkComponents components)
+        throws IOException {
+      // Setting an empty payload since BigQuery transform payload is not actually used by runners
+      // currently.
+      // This can be implemented if runners started actually using the BigQuery transform payload.
+      return FunctionSpec.newBuilder().setUrn(getUrn()).setPayload(ByteString.empty()).build();
+    }
+
+    @Override
+    public Row toConfigRow(TypedRead<?> transform) {
+      Map<String, Object> fieldValues = new HashMap<>();
+
+      if (transform.getJsonTableRef() != null) {
+        fieldValues.put("json_table_ref", transform.getJsonTableRef().get());
+      }
+      if (transform.getQuery() != null) {
+        fieldValues.put("query", transform.getQuery().get());
+      }
+      fieldValues.put("validate", transform.getValidate());
+      fieldValues.put("flatten_results", transform.getFlattenResults());
+      fieldValues.put("use_legacy_sql", transform.getUseLegacySql());
+      fieldValues.put("with_template_compatibility", transform.getWithTemplateCompatibility());
+
+      if (transform.getBigQueryServices() != null) {
+        fieldValues.put("bigquery_services", toByteArray(transform.getBigQueryServices()));
+      }
+      if (transform.getParseFn() != null) {
+        fieldValues.put("parse_fn", toByteArray(transform.getParseFn()));
+      }
+      if (transform.getDatumReaderFactory() != null) {
+        fieldValues.put("datum_reader_factory", toByteArray(transform.getDatumReaderFactory()));
+      }
+      if (transform.getQueryPriority() != null) {
+        fieldValues.put("query_priority", toByteArray(transform.getQueryPriority()));
+      }
+      if (transform.getQueryLocation() != null) {
+        fieldValues.put("query_location", transform.getQueryLocation());
+      }
+      if (transform.getQueryTempDataset() != null) {
+        fieldValues.put("query_temp_dataset", transform.getQueryTempDataset());
+      }
+      if (transform.getMethod() != null) {
+        fieldValues.put("method", toByteArray(transform.getMethod()));
+      }
+      if (transform.getFormat() != null) {
+        fieldValues.put("format", toByteArray(transform.getFormat()));
+      }
+      if (transform.getSelectedFields() != null && !transform.getSelectedFields().get().isEmpty()) {
+        fieldValues.put("selected_fields", transform.getSelectedFields().get());
+      }
+      if (transform.getRowRestriction() != null) {
+        fieldValues.put("row_restriction", transform.getRowRestriction().get());
+      }
+      if (transform.getCoder() != null) {
+        fieldValues.put("coder", toByteArray(transform.getCoder()));
+      }
+      if (transform.getKmsKey() != null) {
+        fieldValues.put("kms_key", transform.getKmsKey());
+      }
+      if (transform.getTypeDescriptor() != null) {
+        fieldValues.put("type_descriptor", toByteArray(transform.getTypeDescriptor()));
+      }
+      if (transform.getToBeamRowFn() != null) {
+        fieldValues.put("to_beam_row_fn", toByteArray(transform.getToBeamRowFn()));
+      }
+      if (transform.getFromBeamRowFn() != null) {
+        fieldValues.put("from_beam_row_fn", toByteArray(transform.getFromBeamRowFn()));
+      }
+      if (transform.getUseAvroLogicalTypes() != null) {
+        fieldValues.put("use_avro_logical_types", transform.getUseAvroLogicalTypes());
+      }
+      fieldValues.put("projection_pushdown_applied", transform.getProjectionPushdownApplied());
+
+      return Row.withSchema(schema).withFieldValues(fieldValues).build();
+    }
+
+    @Override
+    public TypedRead<?> fromConfigRow(Row configRow) {
+      try {
+        BigQueryIO.TypedRead.Builder builder = new AutoValue_BigQueryIO_TypedRead.Builder<>();
+
+        String jsonTableRef = configRow.getString("json_table_ref");
+        if (jsonTableRef != null) {
+          builder = builder.setJsonTableRef(StaticValueProvider.of(jsonTableRef));
+        }
+        String query = configRow.getString("query");
+        if (query != null) {
+          builder = builder.setQuery(StaticValueProvider.of(query));
+        }
+        Boolean validate = configRow.getBoolean("validate");
+        if (validate != null) {
+          builder = builder.setValidate(validate);
+        }
+        Boolean flattenResults = configRow.getBoolean("flatten_results");
+        if (flattenResults != null) {
+          builder = builder.setFlattenResults(flattenResults);
+        }
+        Boolean useLegacySQL = configRow.getBoolean("use_legacy_sql");
+        if (useLegacySQL != null) {
+          builder = builder.setUseLegacySql(useLegacySQL);
+        }
+        Boolean withTemplateCompatibility = configRow.getBoolean("with_template_compatibility");
+        if (withTemplateCompatibility != null) {
+          builder = builder.setWithTemplateCompatibility(withTemplateCompatibility);
+        }
+        byte[] bigqueryServicesBytes = configRow.getBytes("bigquery_services");
+        if (bigqueryServicesBytes != null) {
+          try {
+            builder =
+                builder.setBigQueryServices(
+                    (BigQueryServices) fromByteArray(bigqueryServicesBytes));
+          } catch (InvalidClassException e) {
+            LOG.warn(
+                "Could not use the provided `BigQueryServices` implementation when upgrading."
+                    + "Using the default.");
+            builder.setBigQueryServices(new BigQueryServicesImpl());
+          }
+        }
+        byte[] parseFnBytes = configRow.getBytes("parse_fn");
+        if (parseFnBytes != null) {
+          builder = builder.setParseFn((SerializableFunction) fromByteArray(parseFnBytes));
+        }
+        byte[] datumReaderFactoryBytes = configRow.getBytes("datum_reader_factory");
+        if (datumReaderFactoryBytes != null) {
+          builder =
+              builder.setDatumReaderFactory(
+                  (SerializableFunction) fromByteArray(datumReaderFactoryBytes));
+        }
+        byte[] queryPriorityBytes = configRow.getBytes("query_priority");
+        if (queryPriorityBytes != null) {
+          builder = builder.setQueryPriority((QueryPriority) fromByteArray(queryPriorityBytes));
+        }
+        String queryLocation = configRow.getString("query_location");
+        if (queryLocation != null) {
+          builder = builder.setQueryLocation(queryLocation);
+        }
+        String queryTempDataset = configRow.getString("query_temp_dataset");
+        if (queryTempDataset != null) {
+          builder = builder.setQueryTempDataset(queryTempDataset);
+        }
+        byte[] methodBytes = configRow.getBytes("method");
+        if (methodBytes != null) {
+          builder = builder.setMethod((TypedRead.Method) fromByteArray(methodBytes));
+        }
+        byte[] formatBytes = configRow.getBytes("format");
+        if (methodBytes != null) {
+          builder = builder.setFormat((DataFormat) fromByteArray(formatBytes));
+        }
+        Collection<String> selectedFields = configRow.getArray("selected_fields");
+        if (selectedFields != null && !selectedFields.isEmpty()) {
+          builder.setSelectedFields(StaticValueProvider.of(ImmutableList.of(selectedFields)));
+        }
+        String rowRestriction = configRow.getString("row_restriction");
+        if (rowRestriction != null) {
+          builder = builder.setRowRestriction(StaticValueProvider.of(rowRestriction));
+        }
+        byte[] coderBytes = configRow.getBytes("coder");
+        if (coderBytes != null) {
+          try {
+            builder = builder.setCoder((Coder) fromByteArray(coderBytes));
+          } catch (InvalidClassException e) {
+            LOG.warn(
+                "Could not use the provided `Coder` implementation when upgrading."
+                    + "Using the default.");
+          }
+        }
+        String kmsKey = configRow.getString("kms_key");
+        if (kmsKey != null) {
+          builder = builder.setKmsKey(kmsKey);
+        }
+        byte[] typeDescriptorBytes = configRow.getBytes("type_descriptor");
+        if (typeDescriptorBytes != null) {
+          builder = builder.setTypeDescriptor((TypeDescriptor) fromByteArray(typeDescriptorBytes));
+        }
+        byte[] toBeamRowFnBytes = configRow.getBytes("to_beam_row_fn");
+        if (toBeamRowFnBytes != null) {
+          builder = builder.setToBeamRowFn((ToBeamRowFunction) fromByteArray(toBeamRowFnBytes));
+        }
+        byte[] fromBeamRowFnBytes = configRow.getBytes("from_beam_row_fn");
+        if (fromBeamRowFnBytes != null) {
+          builder =
+              builder.setFromBeamRowFn((FromBeamRowFunction) fromByteArray(fromBeamRowFnBytes));
+        }
+        Boolean useAvroLogicalTypes = configRow.getBoolean("use_avro_logical_types");
+        if (useAvroLogicalTypes != null) {
+          builder = builder.setUseAvroLogicalTypes(useAvroLogicalTypes);
+        }
+        Boolean projectionPushdownApplied = configRow.getBoolean("projection_pushdown_applied");
+        if (projectionPushdownApplied != null) {
+          builder = builder.setProjectionPushdownApplied(projectionPushdownApplied);
+        }
+
+        return builder.build();
+      } catch (InvalidClassException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  @AutoService(TransformPayloadTranslatorRegistrar.class)
+  public static class ReadRegistrar implements TransformPayloadTranslatorRegistrar {
+
+    @Override
+    @SuppressWarnings({
+      "rawtypes",
+    })
+    public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+        getTransformPayloadTranslators() {
+      return ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
+          .put(AutoValue_BigQueryIO_TypedRead.class, new BigQueryIOReadTranslator())
+          .build();
+    }
+  }
+
+  static class BigQueryIOWriteTranslator implements TransformPayloadTranslator<Write<?>> {
+
+    static Schema schema =
+        Schema.builder()
+            .addNullableStringField("json_table_ref")
+            .addNullableByteArrayField("table_function")
+            .addNullableByteArrayField("format_function")
+            .addNullableByteArrayField("format_record_on_failure_function")
+            .addNullableByteArrayField("avro_row_writer_factory")
+            .addNullableByteArrayField("avro_schema_factory")
+            .addNullableBooleanField("use_avro_logical_types")
+            .addNullableByteArrayField("dynamic_destinations")
+            .addNullableStringField("json_schema")
+            .addNullableStringField("json_time_partitioning")
+            .addNullableByteArrayField("clustering")
+            .addNullableByteArrayField("create_disposition")
+            .addNullableByteArrayField("write_disposition")
+            .addNullableArrayField("schema_update_options", FieldType.BYTES)
+            .addNullableStringField("table_description")
+            .addNullableBooleanField("validate")
+            .addNullableByteArrayField("bigquery_services")
+            .addNullableInt32Field("max_files_per_bundle")
+            .addNullableInt64Field("max_file_size")
+            .addNullableInt32Field("num_file_shards")
+            .addNullableInt32Field("num_storage_write_api_streams")
+            .addNullableBooleanField("propagate_successful_storage_api_writes")
+            .addNullableInt32Field("max_files_per_partition")
+            .addNullableInt64Field("max_bytes_per_partition")
+            .addNullableLogicalTypeField("triggering_frequency", new NanosDuration())
+            .addNullableByteArrayField("method")
+            .addNullableStringField("load_job_project_id")
+            .addNullableByteArrayField("failed_insert_retry_policy")
+            .addNullableStringField("custom_gcs_temp_location")
+            .addNullableBooleanField("extended_error_info")
+            .addNullableBooleanField("skip_invalid_rows")
+            .addNullableBooleanField("ignore_unknown_values")
+            .addNullableBooleanField("ignore_insert_ids")
+            .addNullableInt32Field("max_retry_jobs")
+            .addNullableStringField("kms_key")
+            .addNullableArrayField("primary_key", FieldType.STRING)
+            .addNullableByteArrayField("default_missing_value_interpretation")
+            .addNullableBooleanField("optimize_writes")
+            .addNullableBooleanField("use_beam_schema")
+            .addNullableBooleanField("auto_sharding")
+            .addNullableBooleanField("propagate_successful")
+            .addNullableBooleanField("auto_schema_update")
+            .addNullableByteArrayField("write_protos_class")
+            .addNullableBooleanField("direct_write_protos")
+            .addNullableByteArrayField("deterministic_record_id_fn")
+            .addNullableStringField("write_temp_dataset")
+            .addNullableByteArrayField("row_mutation_information_fn")
+            .build();
+
+    public static final String BIGQUERY_WRITE_TRANSFORM_URN =
+        "beam:transform:org.apache.beam:bigquery_write:v1";
+
+    @Override
+    public String getUrn() {
+      return BIGQUERY_WRITE_TRANSFORM_URN;
+    }
+
+    @Override
+    public @Nullable FunctionSpec translate(
+        AppliedPTransform<?, ?, Write<?>> application, SdkComponents components)
+        throws IOException {
+      // Setting an empty payload since BigQuery transform payload is not actually used by runners
+      // currently.
+      // This can be implemented if runners started actually using the Kafka BigQuery payload.
+      return FunctionSpec.newBuilder().setUrn(getUrn()).setPayload(ByteString.empty()).build();
+    }
+
+    @Override
+    public Row toConfigRow(Write<?> transform) {
+
+      Map<String, Object> fieldValues = new HashMap<>();
+
+      if (transform.getJsonTableRef() != null) {
+        fieldValues.put("json_table_ref", transform.getJsonTableRef().get());
+      }
+      if (transform.getTableFunction() != null) {
+        fieldValues.put("table_function", toByteArray(transform.getTableFunction()));
+      }
+      if (transform.getFormatFunction() != null) {
+        fieldValues.put("format_function", toByteArray(transform.getFormatFunction()));
+      }
+      if (transform.getFormatRecordOnFailureFunction() != null) {
+        fieldValues.put(
+            "format_record_on_failure_function",
+            toByteArray(transform.getFormatRecordOnFailureFunction()));
+      }
+      if (transform.getAvroRowWriterFactory() != null) {
+        fieldValues.put(
+            "avro_row_writer_factory", toByteArray(transform.getAvroRowWriterFactory()));
+      }
+      fieldValues.put("use_avro_logical_types", transform.getUseAvroLogicalTypes());
+      if (transform.getDynamicDestinations() != null) {
+        fieldValues.put("dynamic_destinations", toByteArray(transform.getDynamicDestinations()));
+      }
+      if (transform.getSchemaFromView() != null) {
+        // Property 'getSchemaFromView' cannot be used in a portable way across pipelines since it
+        // is bound to PCollections generated for the current pipeline instance.
+        throw new IllegalArgumentException(
+            "BigQueryIO.Write transforms cannot be converted to a "
+                + "portable row based config due to 'withSchemaFromView' property being set. Please "
+                + "retry without setting this property when configuring your transform");
+      }
+      if (transform.getJsonSchema() != null) {
+        fieldValues.put("json_schema", transform.getJsonSchema().get());
+      }
+      if (transform.getJsonTimePartitioning() != null) {
+        fieldValues.put(
+            "json_time_partitioning", toByteArray(transform.getJsonTimePartitioning().get()));
+      }
+      if (transform.getClustering() != null) {
+        fieldValues.put("clustering", toByteArray(transform.getClustering()));
+      }
+      if (transform.getCreateDisposition() != null) {
+        fieldValues.put("create_disposition", toByteArray(transform.getCreateDisposition()));
+      }
+      if (transform.getWriteDisposition() != null) {
+        fieldValues.put("write_disposition", toByteArray(transform.getWriteDisposition()));
+      }
+      if (transform.getSchemaUpdateOptions() != null
+          && !transform.getSchemaUpdateOptions().isEmpty()) {
+        List<byte[]> schemUpdateOptionsData =
+            transform.getSchemaUpdateOptions().stream()
+                .map(option -> toByteArray(option))
+                .collect(Collectors.toList());
+        fieldValues.put("schema_update_options", schemUpdateOptionsData);
+      }
+      if (transform.getTableDescription() != null) {
+        fieldValues.put("table_description", transform.getTableDescription());
+      }
+      fieldValues.put("validate", transform.getValidate());
+      if (transform.getBigQueryServices() != null) {
+        fieldValues.put("bigquery_services", toByteArray(transform.getBigQueryServices()));
+      }
+      if (transform.getMaxFilesPerBundle() != null) {
+        fieldValues.put("max_files_per_bundle", transform.getMaxFilesPerBundle());
+      }
+      if (transform.getMaxFileSize() != null) {
+        fieldValues.put("max_file_size", transform.getMaxFileSize());
+      }
+      fieldValues.put("num_file_shards", transform.getNumFileShards());
+      fieldValues.put("num_storage_write_api_streams", transform.getNumStorageWriteApiStreams());
+      fieldValues.put(
+          "propagate_successful_storage_api_writes",
+          transform.getPropagateSuccessfulStorageApiWrites());
+      fieldValues.put("max_files_per_partition", transform.getMaxFilesPerPartition());
+      fieldValues.put("max_bytes_per_partition", transform.getMaxBytesPerPartition());
+      if (transform.getTriggeringFrequency() != null) {
+        fieldValues.put(
+            "triggering_frequency",
+            Duration.ofMillis(transform.getTriggeringFrequency().getMillis()));
+      }
+      if (transform.getMethod() != null) {
+        fieldValues.put("method", toByteArray(transform.getMethod()));
+      }
+      if (transform.getLoadJobProjectId() != null) {
+        fieldValues.put("load_job_project_id", transform.getLoadJobProjectId());
+      }
+      if (transform.getFailedInsertRetryPolicy() != null) {
+        fieldValues.put(
+            "failed_insert_retry_policy", toByteArray(transform.getFailedInsertRetryPolicy()));
+      }
+      if (transform.getCustomGcsTempLocation() != null) {
+        fieldValues.put("custom_gcs_temp_location", transform.getCustomGcsTempLocation().get());
+      }
+      fieldValues.put("extended_error_info", transform.getExtendedErrorInfo());
+      if (transform.getSkipInvalidRows() != null) {
+        fieldValues.put("skip_invalid_rows", transform.getSkipInvalidRows());
+      }
+      if (transform.getIgnoreUnknownValues() != null) {
+        fieldValues.put("ignore_unknown_values", transform.getIgnoreUnknownValues());
+      }
+      if (transform.getIgnoreInsertIds() != null) {
+        fieldValues.put("ignore_insert_ids", transform.getIgnoreInsertIds());
+      }
+      fieldValues.put("max_retry_jobs", transform.getMaxRetryJobs());
+      if (transform.getPropagateSuccessful() != null) {
+        fieldValues.put("propagate_successful", transform.getPropagateSuccessful());
+      }
+      if (transform.getKmsKey() != null) {
+        fieldValues.put("kms_key", transform.getKmsKey());
+      }
+      if (transform.getPrimaryKey() != null) {
+        fieldValues.put("primary_key", transform.getPrimaryKey());
+      }
+      if (transform.getDefaultMissingValueInterpretation() != null) {
+        fieldValues.put(
+            "default_missing_value_interpretation",
+            toByteArray(transform.getDefaultMissingValueInterpretation()));
+      }
+      if (transform.getOptimizeWrites() != null) {
+        fieldValues.put("optimize_writes", transform.getOptimizeWrites());
+      }
+      if (transform.getUseBeamSchema() != null) {
+        fieldValues.put("use_beam_schema", transform.getUseBeamSchema());
+      }
+      if (transform.getAutoSharding() != null) {
+        fieldValues.put("auto_sharding", transform.getAutoSharding());
+      }
+      if (transform.getAutoSchemaUpdate() != null) {
+        fieldValues.put("auto_schema_update", transform.getAutoSchemaUpdate());
+      }
+      if (transform.getWriteProtosClass() != null) {
+        fieldValues.put("write_protos_class", toByteArray(transform.getWriteProtosClass()));
+      }
+      if (transform.getDirectWriteProtos() != null) {
+        fieldValues.put("direct_write_protos", transform.getDirectWriteProtos());
+      }
+      if (transform.getDeterministicRecordIdFn() != null) {
+        fieldValues.put(
+            "deterministic_record_id_fn", toByteArray(transform.getDeterministicRecordIdFn()));
+      }
+      if (transform.getWriteTempDataset() != null) {
+        fieldValues.put("write_temp_dataset", toByteArray(transform.getDeterministicRecordIdFn()));
+      }
+      if (transform.getRowMutationInformationFn() != null) {
+        fieldValues.put(
+            "row_mutation_information_fn", toByteArray(transform.getRowMutationInformationFn()));
+      }
+
+      return Row.withSchema(schema).withFieldValues(fieldValues).build();
+    }
+
+    @Override
+    public Write<?> fromConfigRow(Row configRow) {
+      try {
+        BigQueryIO.Write.Builder builder = new AutoValue_BigQueryIO_Write.Builder<>();
+
+        String jsonTableRef = configRow.getString("json_table_ref");
+        if (jsonTableRef != null) {
+          builder = builder.setJsonTableRef(StaticValueProvider.of(jsonTableRef));
+        }
+        byte[] tableFunctionBytes = configRow.getBytes("table_function");
+        if (tableFunctionBytes != null) {
+          builder =
+              builder.setTableFunction(
+                  (SerializableFunction<ValueInSingleWindow, TableDestination>)
+                      fromByteArray(tableFunctionBytes));
+        }
+        byte[] formatFunctionBytes = configRow.getBytes("format_function");
+        if (formatFunctionBytes != null) {
+          builder =
+              builder.setFormatFunction(
+                  (SerializableFunction<?, TableRow>) fromByteArray(formatFunctionBytes));
+        }
+        byte[] formatRecordOnFailureFunctionBytes =
+            configRow.getBytes("format_record_on_failure_function");
+        if (tableFunctionBytes != null) {
+          builder =
+              builder.setFormatRecordOnFailureFunction(
+                  (SerializableFunction<?, TableRow>)
+                      fromByteArray(formatRecordOnFailureFunctionBytes));
+        }
+        byte[] avroRowWriterFactoryBytes = configRow.getBytes("avro_row_writer_factory");
+        if (avroRowWriterFactoryBytes != null) {
+          builder =
+              builder.setAvroRowWriterFactory(
+                  (AvroRowWriterFactory) fromByteArray(avroRowWriterFactoryBytes));
+        }
+        byte[] avroSchemaFactoryBytes = configRow.getBytes("avro_schema_factory");
+        if (tableFunctionBytes != null) {
+          builder =
+              builder.setAvroSchemaFactory(
+                  (SerializableFunction) fromByteArray(avroSchemaFactoryBytes));
+        }
+        Boolean useAvroLogicalTypes = configRow.getBoolean("use_avro_logical_types");
+        if (useAvroLogicalTypes != null) {
+          builder = builder.setUseAvroLogicalTypes(useAvroLogicalTypes);
+        }
+        byte[] dynamicDestinationsBytes = configRow.getBytes("dynamic_destinations");
+        if (dynamicDestinationsBytes != null) {
+          builder =
+              builder.setDynamicDestinations(
+                  (DynamicDestinations) fromByteArray(dynamicDestinationsBytes));
+        }
+        String jsonSchema = configRow.getString("json_schema");
+        if (jsonSchema != null) {
+          builder = builder.setJsonSchema(StaticValueProvider.of(jsonSchema));
+        }
+        String jsonTimePartitioning = configRow.getString("json_time_partitioning");
+        if (jsonTimePartitioning != null) {
+          builder = builder.setJsonTimePartitioning(StaticValueProvider.of(jsonTimePartitioning));
+        }
+        byte[] clusteringBytes = configRow.getBytes("clustering");
+        if (clusteringBytes != null) {
+          builder = builder.setClustering((Clustering) fromByteArray(clusteringBytes));
+        }
+        byte[] createDispositionBytes = configRow.getBytes("create_disposition");
+        if (createDispositionBytes != null) {
+          builder =
+              builder.setCreateDisposition(
+                  (CreateDisposition) fromByteArray(createDispositionBytes));
+        }
+        byte[] writeDispositionBytes = configRow.getBytes("write_disposition");
+        if (writeDispositionBytes != null) {
+          builder =
+              builder.setWriteDisposition((WriteDisposition) fromByteArray(writeDispositionBytes));
+        }
+        Collection<byte[]> schemaUpdateOptionsData = configRow.getArray("schema_update_options");
+        if (schemaUpdateOptionsData != null) {
+          Set<SchemaUpdateOption> schemaUpdateOptions =
+              schemaUpdateOptionsData.stream()
+                  .map(
+                      data -> {
+                        try {
+                          return (SchemaUpdateOption) fromByteArray(data);
+                        } catch (InvalidClassException e) {
+                          throw new RuntimeException(e);
+                        }
+                      })
+                  .collect(Collectors.toSet());
+          builder = builder.setSchemaUpdateOptions(schemaUpdateOptions);
+        } else {
+          // This property is not nullable.
+          builder = builder.setSchemaUpdateOptions(Collections.emptySet());
+        }
+        String tableDescription = configRow.getString("table_description");
+        if (tableDescription != null) {
+          builder = builder.setTableDescription(tableDescription);
+        }
+        Boolean validate = configRow.getBoolean("validate");
+        if (validate != null) {
+          builder = builder.setValidate(validate);
+        }
+        byte[] bigqueryServicesBytes = configRow.getBytes("bigquery_services");
+        if (bigqueryServicesBytes != null) {
+          try {
+            builder =
+                builder.setBigQueryServices(
+                    (BigQueryServices) fromByteArray(bigqueryServicesBytes));
+          } catch (InvalidClassException e) {
+            LOG.warn(
+                "Could not use the provided `BigQueryServices` implementation when upgrading."
+                    + "Using the default.");
+            builder.setBigQueryServices(new BigQueryServicesImpl());
+          }
+        }
+        Integer maxFilesPerBundle = configRow.getInt32("max_files_per_bundle");
+        if (maxFilesPerBundle != null) {
+          builder = builder.setMaxFilesPerBundle(maxFilesPerBundle);
+        }
+        Long maxFileSize = configRow.getInt64("max_file_size");
+        if (maxFileSize != null) {
+          builder = builder.setMaxFileSize(maxFileSize);
+        }
+        Integer numFileShards = configRow.getInt32("num_file_shards");
+        if (numFileShards != null) {
+          builder = builder.setNumFileShards(numFileShards);
+        }
+        Integer numStorageWriteApiStreams = configRow.getInt32("num_storage_write_api_streams");
+        if (numStorageWriteApiStreams != null) {
+          builder = builder.setNumStorageWriteApiStreams(numStorageWriteApiStreams);
+        }
+        Boolean propagateSuccessfulStorageApiWrites =
+            configRow.getBoolean("propagate_successful_storage_api_writes");
+        if (propagateSuccessfulStorageApiWrites != null) {
+          builder =
+              builder.setPropagateSuccessfulStorageApiWrites(propagateSuccessfulStorageApiWrites);
+        }
+        Integer maxFilesPerPartition = configRow.getInt32("max_files_per_partition");
+        if (maxFilesPerPartition != null) {
+          builder = builder.setMaxFilesPerPartition(maxFilesPerPartition);
+        }
+        Long maxBytesPerPartition = configRow.getInt64("max_bytes_per_partition");
+        if (maxBytesPerPartition != null) {
+          builder = builder.setMaxBytesPerPartition(maxBytesPerPartition);
+        }
+        Duration triggeringFrequency = configRow.getValue("triggering_frequency");
+        if (triggeringFrequency != null) {
+          builder =
+              builder.setTriggeringFrequency(
+                  org.joda.time.Duration.millis(triggeringFrequency.toMillis()));
+        }
+        byte[] methodBytes = configRow.getBytes("method");
+        if (methodBytes != null) {
+          builder = builder.setMethod((Write.Method) fromByteArray(methodBytes));
+        }
+        String loadJobProjectId = configRow.getString("load_job_project_id");
+        if (loadJobProjectId != null) {
+          builder = builder.setLoadJobProjectId(StaticValueProvider.of(loadJobProjectId));
+        }
+        byte[] failedInsertRetryPolicyBytes = configRow.getBytes("failed_insert_retry_policy");
+        if (failedInsertRetryPolicyBytes != null) {
+          builder =
+              builder.setFailedInsertRetryPolicy(
+                  (InsertRetryPolicy) fromByteArray(failedInsertRetryPolicyBytes));
+        }
+        String customGcsTempLocations = configRow.getString("custom_gcs_temp_location");
+        if (customGcsTempLocations != null) {
+          builder =
+              builder.setCustomGcsTempLocation(StaticValueProvider.of(customGcsTempLocations));
+        }
+        Boolean extendedErrorInfo = configRow.getBoolean("extended_error_info");
+        if (extendedErrorInfo != null) {
+          builder = builder.setExtendedErrorInfo(extendedErrorInfo);
+        }
+        Boolean skipInvalidRows = configRow.getBoolean("skip_invalid_rows");
+        if (skipInvalidRows != null) {
+          builder = builder.setSkipInvalidRows(skipInvalidRows);
+        }
+        Boolean ignoreUnknownValues = configRow.getBoolean("ignore_unknown_values");
+        if (ignoreUnknownValues != null) {
+          builder = builder.setIgnoreUnknownValues(ignoreUnknownValues);
+        }
+        Boolean ignoreInsertIds = configRow.getBoolean("ignore_insert_ids");
+        if (ignoreInsertIds != null) {
+          builder = builder.setIgnoreInsertIds(ignoreInsertIds);
+        }
+        Integer maxRetryJobs = configRow.getInt32("max_retry_jobs");
+        if (maxRetryJobs != null) {
+          builder = builder.setMaxRetryJobs(maxRetryJobs);
+        }
+        String kmsKey = configRow.getString("kms_key");
+        if (kmsKey != null) {
+          builder = builder.setKmsKey(kmsKey);
+        }
+        Collection<String> primaryKey = configRow.getArray("primary_key");
+        if (primaryKey != null && !primaryKey.isEmpty()) {
+          builder = builder.setPrimaryKey(ImmutableList.of(primaryKey));
+        }
+        byte[] defaultMissingValueInterpretationsBytes =
+            configRow.getBytes("default_missing_value_interpretation");
+        if (defaultMissingValueInterpretationsBytes != null) {
+          builder =
+              builder.setDefaultMissingValueInterpretation(
+                  (MissingValueInterpretation)
+                      fromByteArray(defaultMissingValueInterpretationsBytes));
+        }
+        Boolean optimizeWrites = configRow.getBoolean("optimize_writes");
+        if (optimizeWrites != null) {
+          builder = builder.setOptimizeWrites(optimizeWrites);
+        }
+        Boolean useBeamSchema = configRow.getBoolean("use_beam_schema");
+        if (useBeamSchema != null) {
+          builder = builder.setUseBeamSchema(useBeamSchema);
+        }
+        Boolean autoSharding = configRow.getBoolean("auto_sharding");
+        if (autoSharding != null) {
+          builder = builder.setAutoSharding(autoSharding);
+        }
+        Boolean propagateSuccessful = configRow.getBoolean("propagate_successful");
+        if (propagateSuccessful != null) {
+          builder = builder.setPropagateSuccessful(propagateSuccessful);
+        }
+        Boolean autoSchemaUpdate = configRow.getBoolean("auto_schema_update");
+        if (autoSchemaUpdate != null) {
+          builder = builder.setAutoSchemaUpdate(autoSchemaUpdate);
+        }
+        byte[] writeProtosClasses = configRow.getBytes("write_protos_class");
+        if (writeProtosClasses != null) {
+          builder =
+              builder.setWriteProtosClass(
+                  (Class) fromByteArray(defaultMissingValueInterpretationsBytes));
+        }
+        Boolean directWriteProtos = configRow.getBoolean("direct_write_protos");
+        if (directWriteProtos != null) {
+          builder = builder.setDirectWriteProtos(directWriteProtos);
+        }
+        byte[] deterministicRecordIdFnBytes = configRow.getBytes("deterministic_record_id_fn");
+        if (deterministicRecordIdFnBytes != null) {
+          builder =
+              builder.setDeterministicRecordIdFn(
+                  (SerializableFunction) fromByteArray(deterministicRecordIdFnBytes));
+        }
+        String writeTempDataset = configRow.getString("write_temp_dataset");
+        if (writeTempDataset != null) {
+          builder = builder.setWriteTempDataset(writeTempDataset);
+        }
+        byte[] rowMutationInformationFnBytes = configRow.getBytes("row_mutation_information_fn");
+        if (rowMutationInformationFnBytes != null) {
+          builder =
+              builder.setRowMutationInformationFn(
+                  (SerializableFunction) fromByteArray(rowMutationInformationFnBytes));
+        }
+
+        return builder.build();
+      } catch (InvalidClassException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  @AutoService(TransformPayloadTranslatorRegistrar.class)
+  public static class WriteRegistrar implements TransformPayloadTranslatorRegistrar {
+
+    @Override
+    @SuppressWarnings({
+      "rawtypes",
+    })
+    public Map<? extends Class<? extends PTransform>, ? extends TransformPayloadTranslator>
+        getTransformPayloadTranslators() {
+      return ImmutableMap.<Class<? extends PTransform>, TransformPayloadTranslator>builder()
+          .put(AutoValue_BigQueryIO_Write.class, new BigQueryIOWriteTranslator())
+          .build();
+    }
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
index c9c96eb35f3f..a8e1adf643ab 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
@@ -63,6 +63,9 @@ public interface BigQueryServices extends Serializable {
   /** Returns a real, mock, or fake {@link DatasetService}. */
   DatasetService getDatasetService(BigQueryOptions bqOptions);
 
+  /** Returns a real, mock, or fake {@link WriteStreamService}. */
+  WriteStreamService getWriteStreamService(BigQueryOptions bqOptions);
+
   /** Returns a real, mock, or fake {@link StorageClient}. */
   StorageClient getStorageClient(BigQueryOptions bqOptions) throws IOException;
 
@@ -112,7 +115,7 @@ JobStatistics dryRunQuery(
   }
 
   /** An interface to get, create and delete Cloud BigQuery datasets and tables. */
-  public interface DatasetService extends AutoCloseable {
+  interface DatasetService extends AutoCloseable {
 
     // maps the values at
     // https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get#TableMetadataView
@@ -201,7 +204,10 @@ <T> long insertAll(
     /** Patch BigQuery {@link Table} description. */
     Table patchTableDescription(TableReference tableReference, @Nullable String tableDescription)
         throws IOException, InterruptedException;
+  }
 
+  /** An interface to get, create and flush Cloud BigQuery STORAGE API write streams. */
+  interface WriteStreamService extends AutoCloseable {
     /** Create a Write Stream for use with the Storage Write API. */
     WriteStream createWriteStream(String tableUrn, WriteStream.Type type)
         throws IOException, InterruptedException;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index fb288fc1d512..bb3b99f6fcd5 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -197,6 +197,11 @@ public DatasetService getDatasetService(BigQueryOptions options) {
     return new DatasetServiceImpl(options);
   }
 
+  @Override
+  public WriteStreamService getWriteStreamService(BigQueryOptions options) {
+    return new WriteStreamServiceImpl(options);
+  }
+
   @Override
   public StorageClient getStorageClient(BigQueryOptions options) throws IOException {
     return new StorageClientImpl(options);
@@ -563,64 +568,43 @@ public static class DatasetServiceImpl implements DatasetService {
 
     private final ApiErrorExtractor errorExtractor;
     private final Bigquery client;
-    private final @Nullable BigQueryWriteClient newWriteClient;
     private final PipelineOptions options;
     private final long maxRowsPerBatch;
     private final long maxRowBatchSize;
-    private final long storageWriteMaxInflightRequests;
-    private final long storageWriteMaxInflightBytes;
     // aggregate the total time spent in exponential backoff
     private final Counter throttlingMsecs =
         Metrics.counter(DatasetServiceImpl.class, "throttling-msecs");
 
     private @Nullable BoundedExecutorService executor;
-    private final BigQueryIOMetadata bqIOMetadata;
 
     @VisibleForTesting
-    DatasetServiceImpl(
-        Bigquery client, @Nullable BigQueryWriteClient newWriteClient, PipelineOptions options) {
+    DatasetServiceImpl(Bigquery client, PipelineOptions options) {
       BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
       this.errorExtractor = new ApiErrorExtractor();
       this.client = client;
-      this.newWriteClient = newWriteClient;
       this.options = options;
       this.maxRowsPerBatch = bqOptions.getMaxStreamingRowsToBatch();
       this.maxRowBatchSize = bqOptions.getMaxStreamingBatchSize();
-      this.storageWriteMaxInflightRequests = bqOptions.getStorageWriteMaxInflightRequests();
-      this.storageWriteMaxInflightBytes = bqOptions.getStorageWriteMaxInflightBytes();
-      this.bqIOMetadata = BigQueryIOMetadata.create();
       this.executor = null;
     }
 
     @VisibleForTesting
-    DatasetServiceImpl(
-        Bigquery client,
-        BigQueryWriteClient newWriteClient,
-        PipelineOptions options,
-        long maxRowsPerBatch) {
+    DatasetServiceImpl(Bigquery client, PipelineOptions options, long maxRowsPerBatch) {
       BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
       this.errorExtractor = new ApiErrorExtractor();
       this.client = client;
-      this.newWriteClient = newWriteClient;
       this.options = options;
       this.maxRowsPerBatch = maxRowsPerBatch;
       this.maxRowBatchSize = bqOptions.getMaxStreamingBatchSize();
-      this.storageWriteMaxInflightRequests = bqOptions.getStorageWriteMaxInflightRequests();
-      this.storageWriteMaxInflightBytes = bqOptions.getStorageWriteMaxInflightBytes();
-      this.bqIOMetadata = BigQueryIOMetadata.create();
       this.executor = null;
     }
 
     public DatasetServiceImpl(BigQueryOptions bqOptions) {
       this.errorExtractor = new ApiErrorExtractor();
       this.client = newBigQueryClient(bqOptions).build();
-      this.newWriteClient = newBigQueryWriteClient(bqOptions);
       this.options = bqOptions;
       this.maxRowsPerBatch = bqOptions.getMaxStreamingRowsToBatch();
       this.maxRowBatchSize = bqOptions.getMaxStreamingBatchSize();
-      this.storageWriteMaxInflightRequests = bqOptions.getStorageWriteMaxInflightRequests();
-      this.storageWriteMaxInflightBytes = bqOptions.getStorageWriteMaxInflightBytes();
-      this.bqIOMetadata = BigQueryIOMetadata.create();
       this.executor = null;
     }
 
@@ -1344,6 +1328,38 @@ public Table patchTableDescription(
           ALWAYS_RETRY);
     }
 
+    @Override
+    public void close() throws Exception {
+      // Nothing to close
+    }
+  }
+
+  @VisibleForTesting
+  public static class WriteStreamServiceImpl implements WriteStreamService {
+    private final BigQueryWriteClient newWriteClient;
+    private final long storageWriteMaxInflightRequests;
+    private final long storageWriteMaxInflightBytes;
+    private final BigQueryIOMetadata bqIOMetadata;
+    private final PipelineOptions options;
+
+    @VisibleForTesting
+    WriteStreamServiceImpl(BigQueryWriteClient newWriteClient, PipelineOptions options) {
+      BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
+      this.newWriteClient = newWriteClient;
+      this.options = options;
+      this.storageWriteMaxInflightRequests = bqOptions.getStorageWriteMaxInflightRequests();
+      this.storageWriteMaxInflightBytes = bqOptions.getStorageWriteMaxInflightBytes();
+      this.bqIOMetadata = BigQueryIOMetadata.create();
+    }
+
+    public WriteStreamServiceImpl(BigQueryOptions bqOptions) {
+      this.newWriteClient = newBigQueryWriteClient(bqOptions);
+      this.options = bqOptions;
+      this.storageWriteMaxInflightRequests = bqOptions.getStorageWriteMaxInflightRequests();
+      this.storageWriteMaxInflightBytes = bqOptions.getStorageWriteMaxInflightBytes();
+      this.bqIOMetadata = BigQueryIOMetadata.create();
+    }
+
     @Override
     public WriteStream createWriteStream(String tableUrn, WriteStream.Type type)
         throws IOException {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetrics.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetrics.java
index 24323fce6895..a89707c1919f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetrics.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetrics.java
@@ -17,8 +17,10 @@
  */
 package org.apache.beam.sdk.io.gcp.bigquery;
 
+import com.google.auto.value.AutoValue;
 import io.grpc.Status;
 import java.time.Instant;
+import java.util.List;
 import java.util.NavigableMap;
 import java.util.TreeMap;
 import javax.annotation.Nonnull;
@@ -30,6 +32,8 @@
 import org.apache.beam.sdk.metrics.Histogram;
 import org.apache.beam.sdk.metrics.MetricName;
 import org.apache.beam.sdk.util.HistogramData;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Splitter;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
 
 /**
  * Helper class to create perworker metrics for BigQuery Sink stages.
@@ -41,7 +45,7 @@
 public class BigQuerySinkMetrics {
   private static Boolean supportMetricsDeletion = false;
 
-  private static final String METRICS_NAMESPACE = "BigQuerySink";
+  public static final String METRICS_NAMESPACE = "BigQuerySink";
 
   // Status codes
   private static final String UNKNOWN = Status.Code.UNKNOWN.toString();
@@ -79,6 +83,23 @@ enum RowStatus {
   private static final char METRIC_KV_DELIMITER = ':';
   private static final char METRIC_NAME_DELIMITER = '-';
 
+  @AutoValue
+  public abstract static class ParsedMetricName {
+    public abstract String getBaseName();
+
+    public abstract ImmutableMap<String, String> getMetricLabels();
+
+    public static ParsedMetricName create(
+        String baseName, ImmutableMap<String, String> metricLabels) {
+      return new AutoValue_BigQuerySinkMetrics_ParsedMetricName(baseName, metricLabels);
+    }
+
+    public static ParsedMetricName create(String baseName) {
+      ImmutableMap<String, String> emptyMap = ImmutableMap.of();
+      return new AutoValue_BigQuerySinkMetrics_ParsedMetricName(baseName, emptyMap);
+    }
+  }
+
   /**
    * Returns a metric name that merges the baseName with metricLables formatted as.
    *
@@ -94,6 +115,44 @@ private static String createLabeledMetricName(
     return nameBuilder.toString();
   }
 
+  /**
+   * Parse a 'metric name' String that was created with 'createLabeledMetricName'. The input string
+   * should be formatted as.
+   *
+   * <p>'{baseName}-{metricLabelKey1}:{metricLabelVal1};...{metricLabelKeyN}:{metricLabelValN};'
+   *
+   * @param metricName
+   * @return Returns a ParsedMetricName object if the input string is properly formatted. If the
+   *     input string is empty or malformed, returns null.
+   */
+  public static @Nullable ParsedMetricName parseMetricName(String metricName) {
+    if (metricName.isEmpty()) {
+      return null;
+    }
+
+    List<String> metricNameSplit = Splitter.on(METRIC_NAME_DELIMITER).splitToList(metricName);
+    ImmutableMap.Builder<String, String> metricLabelsBuilder = ImmutableMap.builder();
+
+    if (metricNameSplit.size() == 1) {
+      return ParsedMetricName.create(metricNameSplit.get(0));
+    }
+
+    if (metricNameSplit.size() != 2) {
+      return null;
+    }
+
+    List<String> labels = Splitter.on(LABEL_DELIMITER).splitToList(metricNameSplit.get(1));
+    for (String label : labels) {
+      List<String> kv = Splitter.on(METRIC_KV_DELIMITER).splitToList(label);
+      if (kv.size() != 2) {
+        continue;
+      }
+      metricLabelsBuilder.put(kv.get(0), kv.get(1));
+    }
+
+    return ParsedMetricName.create(metricNameSplit.get(0), metricLabelsBuilder.build());
+  }
+
   /**
    * @param method StorageWriteAPI method associated with this metric.
    * @param rpcStatus RPC return status.
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java
index 853a63fca8cc..cbe018ba739c 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFinalizeWritesDoFn.java
@@ -25,7 +25,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.Set;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.WriteStreamService;
 import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.Operation.Context;
 import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType;
 import org.apache.beam.sdk.metrics.Counter;
@@ -64,27 +64,29 @@ class StorageApiFinalizeWritesDoFn extends DoFn<KV<String, String>, Void> {
 
   private Map<String, Collection<String>> commitStreams;
   private final BigQueryServices bqServices;
-  private transient @Nullable DatasetService datasetService;
+  private transient @Nullable WriteStreamService writeStreamService;
 
   public StorageApiFinalizeWritesDoFn(BigQueryServices bqServices) {
     this.bqServices = bqServices;
     this.commitStreams = Maps.newHashMap();
-    this.datasetService = null;
+    this.writeStreamService = null;
   }
 
-  private DatasetService getDatasetService(PipelineOptions pipelineOptions) throws IOException {
-    if (datasetService == null) {
-      datasetService = bqServices.getDatasetService(pipelineOptions.as(BigQueryOptions.class));
+  private WriteStreamService getWriteStreamService(PipelineOptions pipelineOptions)
+      throws IOException {
+    if (writeStreamService == null) {
+      writeStreamService =
+          bqServices.getWriteStreamService(pipelineOptions.as(BigQueryOptions.class));
     }
-    return datasetService;
+    return writeStreamService;
   }
 
   @Teardown
   public void onTeardown() {
     try {
-      if (datasetService != null) {
-        datasetService.close();
-        datasetService = null;
+      if (writeStreamService != null) {
+        writeStreamService.close();
+        writeStreamService = null;
       }
     } catch (Exception e) {
       throw new RuntimeException(e);
@@ -101,7 +103,7 @@ public void process(PipelineOptions pipelineOptions, @Element KV<String, String>
       throws Exception {
     String tableId = element.getKey();
     String streamId = element.getValue();
-    DatasetService datasetService = getDatasetService(pipelineOptions);
+    WriteStreamService writeStreamService = getWriteStreamService(pipelineOptions);
 
     RetryManager<FinalizeWriteStreamResponse, Context<FinalizeWriteStreamResponse>> retryManager =
         new RetryManager<>(
@@ -113,7 +115,7 @@ public void process(PipelineOptions pipelineOptions, @Element KV<String, String>
     retryManager.addOperation(
         c -> {
           finalizeOperationsSent.inc();
-          return datasetService.finalizeWriteStream(streamId);
+          return writeStreamService.finalizeWriteStream(streamId);
         },
         contexts -> {
           RetryManager.Operation.Context<FinalizeWriteStreamResponse> firstContext =
@@ -145,7 +147,7 @@ public void process(PipelineOptions pipelineOptions, @Element KV<String, String>
 
   @FinishBundle
   public void finishBundle(PipelineOptions pipelineOptions) throws Exception {
-    DatasetService datasetService = getDatasetService(pipelineOptions);
+    WriteStreamService writeStreamService = getWriteStreamService(pipelineOptions);
     for (Map.Entry<String, Collection<String>> entry : commitStreams.entrySet()) {
       final String tableId = entry.getKey();
       final Collection<String> streamNames = entry.getValue();
@@ -161,7 +163,7 @@ public void finishBundle(PipelineOptions pipelineOptions) throws Exception {
             Iterable<String> streamsToCommit =
                 Iterables.filter(streamNames, s -> !alreadyCommittedStreams.contains(s));
             batchCommitOperationsSent.inc();
-            return datasetService.commitWriteStreams(tableId, streamsToCommit);
+            return writeStreamService.commitWriteStreams(tableId, streamsToCommit);
           },
           contexts -> {
             RetryManager.Operation.Context<BatchCommitWriteStreamsResponse> firstContext =
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java
index 333a0c0b36bf..dec86c3360b0 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiFlushAndFinalizeDoFn.java
@@ -26,7 +26,7 @@
 import java.time.Instant;
 import java.util.Objects;
 import javax.annotation.Nullable;
-import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.WriteStreamService;
 import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.Operation.Context;
 import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType;
 import org.apache.beam.sdk.io.gcp.bigquery.StorageApiFlushAndFinalizeDoFn.Operation;
@@ -50,7 +50,7 @@ public class StorageApiFlushAndFinalizeDoFn extends DoFn<KV<String, Operation>,
   private static final Logger LOG = LoggerFactory.getLogger(StorageApiFlushAndFinalizeDoFn.class);
 
   private final BigQueryServices bqServices;
-  private transient @Nullable DatasetService datasetService = null;
+  private transient @Nullable WriteStreamService writeStreamService = null;
   private final Counter flushOperationsSent =
       Metrics.counter(StorageApiFlushAndFinalizeDoFn.class, "flushOperationsSent");
   private final Counter flushOperationsSucceeded =
@@ -112,19 +112,21 @@ public StorageApiFlushAndFinalizeDoFn(BigQueryServices bqServices) {
     this.bqServices = bqServices;
   }
 
-  private DatasetService getDatasetService(PipelineOptions pipelineOptions) throws IOException {
-    if (datasetService == null) {
-      datasetService = bqServices.getDatasetService(pipelineOptions.as(BigQueryOptions.class));
+  private WriteStreamService getWriteStreamService(PipelineOptions pipelineOptions)
+      throws IOException {
+    if (writeStreamService == null) {
+      writeStreamService =
+          bqServices.getWriteStreamService(pipelineOptions.as(BigQueryOptions.class));
     }
-    return datasetService;
+    return writeStreamService;
   }
 
   @Teardown
   public void onTeardown() {
     try {
-      if (datasetService != null) {
-        datasetService.close();
-        datasetService = null;
+      if (writeStreamService != null) {
+        writeStreamService.close();
+        writeStreamService = null;
       }
     } catch (Exception e) {
       throw new RuntimeException(e);
@@ -136,7 +138,7 @@ public void process(PipelineOptions pipelineOptions, @Element KV<String, Operati
       throws Exception {
     final String streamId = element.getKey();
     final Operation operation = element.getValue();
-    final DatasetService datasetService = getDatasetService(pipelineOptions);
+    final WriteStreamService datasetService = getWriteStreamService(pipelineOptions);
     // Flush the stream. If the flush offset < 0, that means we only need to finalize.
     long offset = operation.flushOffset;
     if (offset >= 0) {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java
index 13c180a534f1..3c6c73dd0219 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWriteUnshardedRecords.java
@@ -37,6 +37,7 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -56,6 +57,7 @@
 import org.apache.beam.sdk.coders.StringUtf8Coder;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StreamAppendClient;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.WriteStreamService;
 import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType;
 import org.apache.beam.sdk.io.gcp.bigquery.StorageApiDynamicDestinations.MessageConverter;
 import org.apache.beam.sdk.metrics.Counter;
@@ -66,7 +68,9 @@
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.Reshuffle;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.util.Preconditions;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
@@ -271,7 +275,7 @@ class DestinationState {
       private long currentOffset = 0;
       private List<ByteString> pendingMessages;
       private List<org.joda.time.Instant> pendingTimestamps;
-      private transient @Nullable DatasetService maybeDatasetService;
+      private transient @Nullable WriteStreamService maybeWriteStreamService;
       private final Counter recordsAppended =
           Metrics.counter(WriteRecordsDoFn.class, "recordsAppended");
       private final Counter appendFailures =
@@ -298,7 +302,7 @@ public DestinationState(
           String tableUrn,
           String shortTableUrn,
           MessageConverter<ElementT> messageConverter,
-          DatasetService datasetService,
+          WriteStreamService writeStreamService,
           boolean useDefaultStream,
           int streamAppendClientCount,
           boolean usingMultiplexing,
@@ -310,7 +314,7 @@ public DestinationState(
         this.shortTableUrn = shortTableUrn;
         this.pendingMessages = Lists.newArrayList();
         this.pendingTimestamps = Lists.newArrayList();
-        this.maybeDatasetService = datasetService;
+        this.maybeWriteStreamService = writeStreamService;
         this.useDefaultStream = useDefaultStream;
         this.initialTableSchema = messageConverter.getTableSchema();
         this.initialDescriptor = messageConverter.getDescriptor(includeCdcColumns);
@@ -356,7 +360,7 @@ String getOrCreateStreamName() throws Exception {
               () -> {
                 if (!useDefaultStream) {
                   this.streamName =
-                      Preconditions.checkStateNotNull(maybeDatasetService)
+                      Preconditions.checkStateNotNull(maybeWriteStreamService)
                           .createWriteStream(tableUrn, Type.PENDING)
                           .getName();
                   this.currentOffset = 0;
@@ -397,7 +401,7 @@ AppendClientInfo generateClient(@Nullable TableSchema updatedSchema) throws Exce
                   appendClientInfo
                       .get()
                       .withAppendClient(
-                          Preconditions.checkStateNotNull(maybeDatasetService),
+                          Preconditions.checkStateNotNull(maybeWriteStreamService),
                           () -> streamName,
                           usingMultiplexing,
                           defaultMissingValueInterpretation));
@@ -438,7 +442,8 @@ SchemaAndDescriptor getCurrentTableSchema(String stream, @Nullable TableSchema u
               if (autoUpdateSchema) {
                 @Nullable
                 WriteStream writeStream =
-                    Preconditions.checkStateNotNull(maybeDatasetService).getWriteStream(streamName);
+                    Preconditions.checkStateNotNull(maybeWriteStreamService)
+                        .getWriteStream(streamName);
                 if (writeStream != null && writeStream.hasTableSchema()) {
                   TableSchema updatedFromStream = writeStream.getTableSchema();
                   currentSchema.set(updatedFromStream);
@@ -665,10 +670,10 @@ long flush(
                   BigQuerySinkMetrics.throwableToGRPCCodeString(failedContext.getError());
 
               if (failedContext.getError() != null
-                  && failedContext.getError() instanceof Exceptions.AppendSerializtionError) {
-                Exceptions.AppendSerializtionError error =
+                  && failedContext.getError() instanceof Exceptions.AppendSerializationError) {
+                Exceptions.AppendSerializationError error =
                     Preconditions.checkStateNotNull(
-                        (Exceptions.AppendSerializtionError) failedContext.getError());
+                        (Exceptions.AppendSerializationError) failedContext.getError());
 
                 Set<Integer> failedRowIndices = error.getRowIndexToErrorMessage().keySet();
                 for (int failedIndex : failedRowIndices) {
@@ -870,6 +875,7 @@ void postFlush() {
     private @Nullable Map<DestinationT, DestinationState> destinations = Maps.newHashMap();
     private final TwoLevelMessageConverterCache<DestinationT, ElementT> messageConverters;
     private transient @Nullable DatasetService maybeDatasetService;
+    private transient @Nullable WriteStreamService maybeWriteStreamService;
     private int numPendingRecords = 0;
     private int numPendingRecordBytes = 0;
     private final int flushThresholdBytes;
@@ -977,6 +983,14 @@ private DatasetService initializeDatasetService(PipelineOptions pipelineOptions)
       return maybeDatasetService;
     }
 
+    private WriteStreamService initializeWriteStreamService(PipelineOptions pipelineOptions) {
+      if (maybeWriteStreamService == null) {
+        maybeWriteStreamService =
+            bqServices.getWriteStreamService(pipelineOptions.as(BigQueryOptions.class));
+      }
+      return maybeWriteStreamService;
+    }
+
     @StartBundle
     public void startBundle() throws IOException {
       destinations = Maps.newHashMap();
@@ -989,6 +1003,7 @@ DestinationState createDestinationState(
         DestinationT destination,
         boolean useCdc,
         DatasetService datasetService,
+        WriteStreamService writeStreamService,
         BigQueryOptions bigQueryOptions) {
       TableDestination tableDestination1 = dynamicDestinations.getTable(destination);
       checkArgument(
@@ -1019,7 +1034,7 @@ DestinationState createDestinationState(
             tableDestination1.getTableUrn(bigQueryOptions),
             tableDestination1.getShortTableUrn(),
             messageConverter,
-            datasetService,
+            writeStreamService,
             useDefaultStream,
             streamAppendClientCount,
             bigQueryOptions.getUseStorageApiConnectionPool(),
@@ -1040,6 +1055,8 @@ public void process(
         MultiOutputReceiver o)
         throws Exception {
       DatasetService initializedDatasetService = initializeDatasetService(pipelineOptions);
+      WriteStreamService initializedWriteStreamService =
+          initializeWriteStreamService(pipelineOptions);
       dynamicDestinations.setSideInputAccessorFromProcessContext(c);
       DestinationState state =
           Preconditions.checkStateNotNull(destinations)
@@ -1051,6 +1068,7 @@ public void process(
                           destination,
                           usesCdc,
                           initializedDatasetService,
+                          initializedWriteStreamService,
                           pipelineOptions.as(BigQueryOptions.class)));
 
       OutputReceiver<BigQueryStorageApiInsertError> failedRowsReceiver = o.get(failedRowsTag);
@@ -1077,6 +1095,15 @@ public void outputWithTimestamp(
                 BigQueryStorageApiInsertError output, org.joda.time.Instant timestamp) {
               context.output(failedRowsTag, output, timestamp, GlobalWindow.INSTANCE);
             }
+
+            @Override
+            public void outputWindowedValue(
+                BigQueryStorageApiInsertError output,
+                org.joda.time.Instant timestamp,
+                Collection<? extends BoundedWindow> windows,
+                PaneInfo paneInfo) {
+              throw new UnsupportedOperationException("outputWindowedValue not supported");
+            }
           };
       @Nullable OutputReceiver<TableRow> successfulRowsReceiver = null;
       if (successfulRowsTag != null) {
@@ -1091,6 +1118,15 @@ public void output(TableRow output) {
               public void outputWithTimestamp(TableRow output, org.joda.time.Instant timestamp) {
                 context.output(successfulRowsTag, output, timestamp, GlobalWindow.INSTANCE);
               }
+
+              @Override
+              public void outputWindowedValue(
+                  TableRow output,
+                  org.joda.time.Instant timestamp,
+                  Collection<? extends BoundedWindow> windows,
+                  PaneInfo paneInfo) {
+                throw new UnsupportedOperationException("outputWindowedValue not supported");
+              }
             };
       }
 
@@ -1116,6 +1152,10 @@ public void outputWithTimestamp(TableRow output, org.joda.time.Instant timestamp
     public void teardown() {
       destinations = null;
       try {
+        if (maybeWriteStreamService != null) {
+          maybeWriteStreamService.close();
+          maybeWriteStreamService = null;
+        }
         if (maybeDatasetService != null) {
           maybeDatasetService.close();
           maybeDatasetService = null;
@@ -1124,5 +1164,10 @@ public void teardown() {
         throw new RuntimeException(e);
       }
     }
+
+    @Override
+    public Duration getAllowedTimestampSkew() {
+      return Duration.millis(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis());
+    }
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java
index d3042984638f..0f9b07d0c40f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiWritesShardedRecords.java
@@ -57,6 +57,7 @@
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StreamAppendClient;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.WriteStreamService;
 import org.apache.beam.sdk.io.gcp.bigquery.RetryManager.RetryType;
 import org.apache.beam.sdk.io.gcp.bigquery.StorageApiFlushAndFinalizeDoFn.Operation;
 import org.apache.beam.sdk.metrics.Counter;
@@ -319,6 +320,8 @@ class WriteRecordsDoFn
 
     private transient @Nullable DatasetService datasetServiceInternal = null;
 
+    private transient @Nullable WriteStreamService writeStreamServiceInternal = null;
+
     // Stores the current stream for this key.
     @StateId("streamName")
     private final StateSpec<ValueState<String>> streamNameSpec = StateSpecs.value();
@@ -358,7 +361,7 @@ String getOrCreateStream(
         ValueState<String> streamName,
         ValueState<Long> streamOffset,
         Timer streamIdleTimer,
-        DatasetService datasetService,
+        WriteStreamService writeStreamService,
         Callable<Boolean> tryCreateTable) {
       try {
         final @Nullable String streamValue = streamName.read();
@@ -367,7 +370,7 @@ String getOrCreateStream(
           // In a buffered stream, data is only visible up to the offset to which it was flushed.
           CreateTableHelpers.createTableWrapper(
               () -> {
-                stream.set(datasetService.createWriteStream(tableId, Type.BUFFERED).getName());
+                stream.set(writeStreamService.createWriteStream(tableId, Type.BUFFERED).getName());
                 return null;
               },
               tryCreateTable);
@@ -395,9 +398,22 @@ private DatasetService getDatasetService(PipelineOptions pipelineOptions) throws
       return datasetServiceInternal;
     }
 
+    private WriteStreamService getWriteStreamService(PipelineOptions pipelineOptions)
+        throws IOException {
+      if (writeStreamServiceInternal == null) {
+        writeStreamServiceInternal =
+            bqServices.getWriteStreamService(pipelineOptions.as(BigQueryOptions.class));
+      }
+      return writeStreamServiceInternal;
+    }
+
     @Teardown
     public void onTeardown() {
       try {
+        if (writeStreamServiceInternal != null) {
+          writeStreamServiceInternal.close();
+          writeStreamServiceInternal = null;
+        }
         if (datasetServiceInternal != null) {
           datasetServiceInternal.close();
           datasetServiceInternal = null;
@@ -442,6 +458,7 @@ public void process(
       final String tableId = tableDestination.getTableUrn(bigQueryOptions);
       final String shortTableId = tableDestination.getShortTableUrn();
       final DatasetService datasetService = getDatasetService(pipelineOptions);
+      final WriteStreamService writeStreamService = getWriteStreamService(pipelineOptions);
 
       Coder<DestinationT> destinationCoder = dynamicDestinations.getDestinationCoder();
       Callable<Boolean> tryCreateTable =
@@ -462,7 +479,7 @@ public void process(
       Supplier<String> getOrCreateStream =
           () ->
               getOrCreateStream(
-                  tableId, streamName, streamOffset, idleTimer, datasetService, tryCreateTable);
+                  tableId, streamName, streamOffset, idleTimer, writeStreamService, tryCreateTable);
       Callable<AppendClientInfo> getAppendClientInfo =
           () -> {
             @Nullable TableSchema tableSchema;
@@ -500,7 +517,7 @@ public void process(
                                   client.close();
                                 }))
                     .withAppendClient(
-                        datasetService,
+                        writeStreamService,
                         getOrCreateStream,
                         false,
                         defaultMissingValueInterpretation);
@@ -569,7 +586,7 @@ public void process(
                   appendClientInfo
                       .get()
                       .withAppendClient(
-                          datasetService,
+                          writeStreamService,
                           getOrCreateStream,
                           false,
                           defaultMissingValueInterpretation));
@@ -618,7 +635,7 @@ public void process(
                   appendClientInfo
                       .get()
                       .withAppendClient(
-                          datasetService,
+                          writeStreamService,
                           getOrCreateStream,
                           false,
                           defaultMissingValueInterpretation));
@@ -645,10 +662,10 @@ public void process(
             // failedInserts
             // PCollection, and retry with the remaining rows.
             if (failedContext.getError() != null
-                && failedContext.getError() instanceof Exceptions.AppendSerializtionError) {
-              Exceptions.AppendSerializtionError error =
+                && failedContext.getError() instanceof Exceptions.AppendSerializationError) {
+              Exceptions.AppendSerializationError error =
                   Preconditions.checkArgumentNotNull(
-                      (Exceptions.AppendSerializtionError) failedContext.getError());
+                      (Exceptions.AppendSerializationError) failedContext.getError());
 
               Set<Integer> failedRowIndices = error.getRowIndexToErrorMessage().keySet();
               for (int failedIndex : failedRowIndices) {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
index 39e6fd7c809d..8c4edd2244b4 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/providers/BigQueryStorageWriteApiSchemaTransformProvider.java
@@ -98,6 +98,16 @@ public String identifier() {
     return String.format("beam:schematransform:org.apache.beam:bigquery_storage_write:v2");
   }
 
+  @Override
+  public String description() {
+    return String.format(
+        "Writes data to BigQuery using the Storage Write API (https://cloud.google.com/bigquery/docs/write-api)."
+            + "\n\nThis expects a single PCollection of Beam Rows and outputs two dead-letter queues (DLQ) that "
+            + "contain failed rows. The first DLQ has tag [%s] and contains the failed rows. The second DLQ has "
+            + "tag [%s] and contains failed rows and along with their respective errors.",
+        FAILED_ROWS_TAG, FAILED_ROWS_WITH_ERRORS_TAG);
+  }
+
   @Override
   public List<String> inputCollectionNames() {
     return Collections.singletonList(INPUT_ROWS_TAG);
@@ -338,10 +348,13 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
       if (inputRows.isBounded() == IsBounded.UNBOUNDED) {
         Long triggeringFrequency = configuration.getTriggeringFrequencySeconds();
         Boolean autoSharding = configuration.getAutoSharding();
-        Integer numStreams = configuration.getNumStreams();
+        int numStreams = configuration.getNumStreams() == null ? 0 : configuration.getNumStreams();
+        boolean useAtLeastOnceSemantics =
+            configuration.getUseAtLeastOnceSemantics() == null
+                ? false
+                : configuration.getUseAtLeastOnceSemantics();
         // Triggering frequency is only applicable for exactly-once
-        if (configuration.getUseAtLeastOnceSemantics() == null
-            || !configuration.getUseAtLeastOnceSemantics()) {
+        if (!useAtLeastOnceSemantics) {
           write =
               write.withTriggeringFrequency(
                   (triggeringFrequency == null || triggeringFrequency <= 0)
@@ -349,7 +362,7 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
                       : Duration.standardSeconds(triggeringFrequency));
         }
         // set num streams if specified, otherwise default to autoSharding
-        if (numStreams != null && numStreams > 0) {
+        if (numStreams > 0) {
           write = write.withNumStorageWriteApiStreams(numStreams);
         } else if (autoSharding == null || autoSharding) {
           write = write.withAutoSharding();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index ad978e95016a..82c2d3142482 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -60,6 +60,7 @@
 import org.apache.beam.sdk.io.range.ByteKey;
 import org.apache.beam.sdk.io.range.ByteKeyRange;
 import org.apache.beam.sdk.io.range.ByteKeyRangeTracker;
+import org.apache.beam.sdk.options.ExperimentalOptions;
 import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.transforms.DoFn;
@@ -180,6 +181,27 @@
  *         .withBatchElements(100)); // every batch will have 100 elements
  * }</pre>
  *
+ * <p>Configure timeout for writes:
+ *
+ * <pre>{@code
+ * // Let each attempt run for 1 second, retry if the attempt failed.
+ * // Give up after the request is retried for 60 seconds.
+ * Duration attemptTimeout = Duration.millis(1000);
+ * Duration operationTimeout = Duration.millis(60 * 1000);
+ * data.apply("write",
+ *     BigtableIO.write()
+ *         .withProjectId("project")
+ *         .withInstanceId("instance")
+ *         .withTableId("table")
+ *         .withAttemptTimeout(attemptTimeout)
+ *         .withOperationTimeout(operationTimeout));
+ * }</pre>
+ *
+ * <p>You can also limit the wait time in the finish bundle step by setting the
+ * bigtable_writer_wait_timeout_ms experimental flag when you run the pipeline. For example,
+ * --experiments=bigtable_writer_wait_timeout_ms=60000 will limit the wait time in finish bundle to
+ * be 10 minutes.
+ *
  * <p>Optionally, BigtableIO.write() may be configured to emit {@link BigtableWriteResult} elements
  * after each group of inputs is written to Bigtable. These can be used to then trigger user code
  * after writes have completed. See {@link org.apache.beam.sdk.transforms.Wait} for details on the
@@ -1118,6 +1140,8 @@ public static class WriteWithResults
       extends PTransform<
           PCollection<KV<ByteString, Iterable<Mutation>>>, PCollection<BigtableWriteResult>> {
 
+    private static final String BIGTABLE_WRITER_WAIT_TIMEOUT_MS = "bigtable_writer_wait_timeout_ms";
+
     private final BigtableConfig bigtableConfig;
     private final BigtableWriteOptions bigtableWriteOptions;
 
@@ -1138,8 +1162,23 @@ public PCollection<BigtableWriteResult> expand(
       bigtableConfig.validate();
       bigtableWriteOptions.validate();
 
+      // Get experimental flag and set on BigtableWriteOptions
+      PipelineOptions pipelineOptions = input.getPipeline().getOptions();
+      String closeWaitTimeoutStr =
+          ExperimentalOptions.getExperimentValue(pipelineOptions, BIGTABLE_WRITER_WAIT_TIMEOUT_MS);
+      Duration closeWaitTimeout = null;
+      if (closeWaitTimeoutStr != null) {
+        long closeWaitTimeoutMs = Long.parseLong(closeWaitTimeoutStr);
+        checkState(closeWaitTimeoutMs > 0, "Close wait timeout must be positive");
+        closeWaitTimeout = Duration.millis(closeWaitTimeoutMs);
+      }
+
       return input.apply(
-          ParDo.of(new BigtableWriterFn(factory, bigtableConfig, bigtableWriteOptions)));
+          ParDo.of(
+              new BigtableWriterFn(
+                  factory,
+                  bigtableConfig,
+                  bigtableWriteOptions.toBuilder().setCloseWaitTimeout(closeWaitTimeout).build())));
     }
 
     @Override
@@ -1195,6 +1234,7 @@ private static class BigtableWriterFn
       this.writeOptions = writeOptions;
       this.failures = new ConcurrentLinkedQueue<>();
       this.id = factory.newId();
+      LOG.debug("Created Bigtable Write Fn with writeOptions {} ", writeOptions);
     }
 
     @StartBundle
@@ -1205,7 +1245,7 @@ public void startBundle(StartBundleContext c) throws IOException {
       if (bigtableWriter == null) {
         serviceEntry =
             factory.getServiceForWriting(id, config, writeOptions, c.getPipelineOptions());
-        bigtableWriter = serviceEntry.getService().openForWriting(writeOptions.getTableId().get());
+        bigtableWriter = serviceEntry.getService().openForWriting(writeOptions);
       }
     }
 
@@ -1227,27 +1267,26 @@ public void processElement(ProcessContext c, BoundedWindow window) throws Except
 
     @FinishBundle
     public void finishBundle(FinishBundleContext c) throws Exception {
-      bigtableWriter.flush();
-      checkForFailures();
-      LOG.debug("Wrote {} records", recordsWritten);
+      try {
+        if (bigtableWriter != null) {
+          bigtableWriter.close();
+          bigtableWriter = null;
+        }
 
-      for (Map.Entry<BoundedWindow, Long> entry : seenWindows.entrySet()) {
-        c.output(
-            BigtableWriteResult.create(entry.getValue()),
-            entry.getKey().maxTimestamp(),
-            entry.getKey());
-      }
-    }
+        checkForFailures();
+        LOG.debug("Wrote {} records", recordsWritten);
 
-    @Teardown
-    public void tearDown() throws Exception {
-      if (bigtableWriter != null) {
-        bigtableWriter.close();
-        bigtableWriter = null;
-      }
-      if (serviceEntry != null) {
-        serviceEntry.close();
-        serviceEntry = null;
+        for (Map.Entry<BoundedWindow, Long> entry : seenWindows.entrySet()) {
+          c.output(
+              BigtableWriteResult.create(entry.getValue()),
+              entry.getKey().maxTimestamp(),
+              entry.getKey());
+        }
+      } finally {
+        if (serviceEntry != null) {
+          serviceEntry.close();
+          serviceEntry = null;
+        }
       }
     }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java
index c0f88331bd6e..b529d6530ef2 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableService.java
@@ -45,13 +45,6 @@ interface Writer {
     CompletionStage<MutateRowResponse> writeRecord(KV<ByteString, Iterable<Mutation>> record)
         throws IOException;
 
-    /**
-     * Flushes the writer.
-     *
-     * @throws IOException if any writes did not succeed
-     */
-    void flush() throws IOException;
-
     /**
      * Closes the writer.
      *
@@ -88,7 +81,7 @@ interface Reader {
   Reader createReader(BigtableSource source) throws IOException;
 
   /** Returns a {@link Writer} that will write to the specified table. */
-  Writer openForWriting(String tableId) throws IOException;
+  Writer openForWriting(BigtableWriteOptions writeOptions) throws IOException;
 
   /**
    * Returns a set of row keys sampled from the underlying table. These contain information about
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
index 6e98b7e7866a..0fb0b6a88efa 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImpl.java
@@ -19,7 +19,9 @@
 
 import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.MoreExecutors.directExecutor;
 
+import com.google.api.core.ApiFuture;
 import com.google.api.gax.batching.Batcher;
+import com.google.api.gax.batching.BatchingException;
 import com.google.api.gax.grpc.GrpcCallContext;
 import com.google.api.gax.retrying.RetrySettings;
 import com.google.api.gax.rpc.ResponseObserver;
@@ -61,6 +63,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import org.apache.beam.runners.core.metrics.GcpResourceIdentifiers;
 import org.apache.beam.runners.core.metrics.MonitoringInfoConstants;
 import org.apache.beam.runners.core.metrics.ServiceCallMetric;
@@ -106,8 +109,8 @@ class BigtableServiceImpl implements BigtableService {
     RetrySettings retry = settings.getStubSettings().readRowsSettings().getRetrySettings();
     this.readAttemptTimeout = Duration.millis(retry.getInitialRpcTimeout().toMillis());
     this.readOperationTimeout = Duration.millis(retry.getTotalTimeout().toMillis());
-    LOG.info("Started Bigtable service with settings {}", settings);
     this.client = BigtableDataClient.create(settings);
+    LOG.info("Started Bigtable service with settings {}", settings);
   }
 
   private final BigtableDataClient client;
@@ -119,8 +122,13 @@ class BigtableServiceImpl implements BigtableService {
   private final Duration readOperationTimeout;
 
   @Override
-  public BigtableWriterImpl openForWriting(String tableId) {
-    return new BigtableWriterImpl(client, projectId, instanceId, tableId);
+  public BigtableWriterImpl openForWriting(BigtableWriteOptions writeOptions) {
+    return new BigtableWriterImpl(
+        client,
+        projectId,
+        instanceId,
+        writeOptions.getTableId().get(),
+        writeOptions.getCloseWaitTimeout());
   }
 
   @VisibleForTesting
@@ -331,11 +339,16 @@ public boolean start() throws IOException {
 
     @Override
     public boolean advance() throws IOException {
+      if (future != null && future.isDone()) {
+        // Add rows from the future to the buffer and reset the future
+        // so we can do prefetching
+        consumeReadRowsFuture();
+      }
       if (buffer.size() < refillSegmentWaterMark && future == null) {
         future = fetchNextSegment();
       }
       if (buffer.isEmpty() && future != null) {
-        waitReadRowsFuture();
+        consumeReadRowsFuture();
       }
       currentRow = buffer.poll();
       return currentRow != null;
@@ -399,7 +412,7 @@ public void onComplete() {
       return future;
     }
 
-    private void waitReadRowsFuture() throws IOException {
+    private void consumeReadRowsFuture() throws IOException {
       try {
         UpstreamResults r = future.get();
         buffer.addAll(r.rows);
@@ -470,50 +483,55 @@ static class BigtableWriterImpl implements Writer {
     private String projectId;
     private String instanceId;
     private String tableId;
+    private Duration closeWaitTimeout;
 
     private Distribution bulkSize = Metrics.distribution("BigTable-" + tableId, "batchSize");
     private Distribution latency = Metrics.distribution("BigTable-" + tableId, "batchLatencyMs");
 
     BigtableWriterImpl(
-        BigtableDataClient client, String projectId, String instanceId, String tableId) {
+        BigtableDataClient client,
+        String projectId,
+        String instanceId,
+        String tableId,
+        Duration closeWaitTimeout) {
       this.projectId = projectId;
       this.instanceId = instanceId;
       this.tableId = tableId;
+      this.closeWaitTimeout = closeWaitTimeout;
       this.bulkMutation = client.newBulkMutationBatcher(tableId);
     }
 
-    @Override
-    public void flush() throws IOException {
-      if (bulkMutation != null) {
-        try {
-          stopwatch.start();
-          bulkMutation.flush();
-          bulkSize.update(outstandingMutations);
-          outstandingMutations = 0;
-          stopwatch.stop();
-          latency.update(stopwatch.elapsed(TimeUnit.MILLISECONDS));
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-          // We fail since flush() operation was interrupted.
-          throw new IOException(e);
-        }
-      }
-    }
-
     @Override
     public void close() throws IOException {
       if (bulkMutation != null) {
         try {
           stopwatch.start();
-          bulkMutation.flush();
-          bulkMutation.close();
+          // closeAsync will send any remaining elements in the batch.
+          // If the experimental close wait timeout flag is set,
+          // set a timeout waiting for the future.
+          ApiFuture<Void> future = bulkMutation.closeAsync();
+          if (Duration.ZERO.isShorterThan(closeWaitTimeout)) {
+            future.get(closeWaitTimeout.getMillis(), TimeUnit.MILLISECONDS);
+          } else {
+            future.get();
+          }
           bulkSize.update(outstandingMutations);
           outstandingMutations = 0;
           stopwatch.stop();
           latency.update(stopwatch.elapsed(TimeUnit.MILLISECONDS));
+        } catch (BatchingException e) {
+          // Ignore batching failures because element failures are tracked as is in
+          // BigtableIOWriteFn.
+          // TODO: Bigtable client already tracks BatchingExceptions, use BatchingExceptions
+          // instead of tracking them separately in BigtableIOWriteFn.
+        } catch (TimeoutException e) {
+          // We fail because future.get() timed out
+          throw new IOException("BulkMutation took too long to close", e);
+        } catch (ExecutionException e) {
+          throw new IOException("Failed to close batch", e.getCause());
         } catch (InterruptedException e) {
           Thread.currentThread().interrupt();
-          // We fail since flush() operation was interrupted.
+          // We fail since close() operation was interrupted.
           throw new IOException(e);
         }
         bulkMutation = null;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java
index 05e0e915f12a..a63cc575809b 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableWriteOptions.java
@@ -60,6 +60,9 @@ abstract class BigtableWriteOptions implements Serializable {
   /** Returns true if batch write flow control is enabled. Otherwise return false. */
   abstract @Nullable Boolean getFlowControl();
 
+  /** Returns the time to wait when closing the writer. */
+  abstract @Nullable Duration getCloseWaitTimeout();
+
   abstract Builder toBuilder();
 
   static Builder builder() {
@@ -87,6 +90,8 @@ abstract static class Builder {
 
     abstract Builder setFlowControl(boolean enableFlowControl);
 
+    abstract Builder setCloseWaitTimeout(Duration timeout);
+
     abstract BigtableWriteOptions build();
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
index cdd003abae22..b94186b9893d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java
@@ -126,8 +126,9 @@
  * <p>To read a {@link PCollection} from a query to Cloud Datastore, use {@link DatastoreV1#read}
  * and its methods {@link DatastoreV1.Read#withProjectId} and {@link DatastoreV1.Read#withQuery} to
  * specify the project to query and the query to read from. You can optionally provide a namespace
- * to query within using {@link DatastoreV1.Read#withNamespace}. You could also optionally specify
- * how many splits you want for the query using {@link DatastoreV1.Read#withNumQuerySplits}.
+ * to query within using {@link DatastoreV1.Read#withDatabase} or {@link
+ * DatastoreV1.Read#withNamespace}. You could also optionally specify how many splits you want for
+ * the query using {@link DatastoreV1.Read#withNumQuerySplits}.
  *
  * <p>For example:
  *
@@ -135,12 +136,14 @@
  * // Read a query from Datastore
  * PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create();
  * Query query = ...;
+ * String databaseId = "...";
  * String projectId = "...";
  *
  * Pipeline p = Pipeline.create(options);
  * PCollection<Entity> entities = p.apply(
  *     DatastoreIO.v1().read()
  *         .withProjectId(projectId)
+ *         .withDatabaseId(databaseId)
  *         .withQuery(query));
  * }</pre>
  *
@@ -156,7 +159,7 @@
  *
  * <pre>{@code
  * PCollection<Entity> entities = ...;
- * entities.apply(DatastoreIO.v1().write().withProjectId(projectId));
+ * entities.apply(DatastoreIO.v1().write().withProjectId(projectId).withDatabaseId(databaseId));
  * p.run();
  * }</pre>
  *
@@ -165,7 +168,7 @@
  *
  * <pre>{@code
  * PCollection<Entity> entities = ...;
- * entities.apply(DatastoreIO.v1().deleteEntity().withProjectId(projectId));
+ * entities.apply(DatastoreIO.v1().deleteEntity().withProjectId(projectId).withDatabaseId(databaseId));
  * p.run();
  * }</pre>
  *
@@ -174,7 +177,7 @@
  *
  * <pre>{@code
  * PCollection<Entity> entities = ...;
- * entities.apply(DatastoreIO.v1().deleteKey().withProjectId(projectId));
+ * entities.apply(DatastoreIO.v1().deleteKey().withProjectId(projectId).withDatabaseId(databaseId));
  * p.run();
  * }</pre>
  *
@@ -275,6 +278,9 @@ public class DatastoreV1 {
           Code.PERMISSION_DENIED,
           Code.UNAUTHENTICATED);
 
+  /** Database ID for the default database. */
+  private static final String DEFAULT_DATABASE = "";
+
   /**
    * Returns an empty {@link DatastoreV1.Read} builder. Configure the source {@code projectId},
    * {@code query}, and optionally {@code namespace} and {@code numQuerySplits} using {@link
@@ -313,6 +319,8 @@ public abstract static class Read extends PTransform<PBegin, PCollection<Entity>
 
     public abstract @Nullable ValueProvider<String> getProjectId();
 
+    public abstract @Nullable ValueProvider<String> getDatabaseId();
+
     public abstract @Nullable Query getQuery();
 
     public abstract @Nullable ValueProvider<String> getLiteralGqlQuery();
@@ -334,6 +342,8 @@ public abstract static class Read extends PTransform<PBegin, PCollection<Entity>
     abstract static class Builder {
       abstract Builder setProjectId(ValueProvider<String> projectId);
 
+      abstract Builder setDatabaseId(ValueProvider<String> databaseId);
+
       abstract Builder setQuery(Query query);
 
       abstract Builder setLiteralGqlQuery(ValueProvider<String> literalGqlQuery);
@@ -354,10 +364,16 @@ abstract static class Builder {
      * size from Cloud Datastore.
      */
     static int getEstimatedNumSplits(
-        Datastore datastore, Query query, @Nullable String namespace, @Nullable Instant readTime) {
+        Datastore datastore,
+        String projectId,
+        String databaseId,
+        Query query,
+        @Nullable String namespace,
+        @Nullable Instant readTime) {
       int numSplits;
       try {
-        long estimatedSizeBytes = getEstimatedSizeBytes(datastore, query, namespace, readTime);
+        long estimatedSizeBytes =
+            getEstimatedSizeBytes(datastore, projectId, databaseId, query, namespace, readTime);
         LOG.info("Estimated size bytes for the query is: {}", estimatedSizeBytes);
         numSplits =
             (int)
@@ -378,7 +394,11 @@ static int getEstimatedNumSplits(
      * table.
      */
     private static long queryLatestStatisticsTimestamp(
-        Datastore datastore, @Nullable String namespace, @Nullable Instant readTime)
+        Datastore datastore,
+        String projectId,
+        String databaseId,
+        @Nullable String namespace,
+        @Nullable Instant readTime)
         throws DatastoreException {
       Query.Builder query = Query.newBuilder();
       // Note: namespace either being null or empty represents the default namespace, in which
@@ -390,7 +410,8 @@ private static long queryLatestStatisticsTimestamp(
       }
       query.addOrder(makeOrder("timestamp", DESCENDING));
       query.setLimit(Int32Value.newBuilder().setValue(1));
-      RunQueryRequest request = makeRequest(query.build(), namespace, readTime);
+      RunQueryRequest request =
+          makeRequest(projectId, databaseId, query.build(), namespace, readTime);
 
       RunQueryResponse response = datastore.runQuery(request);
       QueryResultBatch batch = response.getBatch();
@@ -406,9 +427,15 @@ private static long queryLatestStatisticsTimestamp(
      * readTime specified, the latest statistics at or before readTime is retrieved.
      */
     private static Entity getLatestTableStats(
-        String ourKind, @Nullable String namespace, Datastore datastore, @Nullable Instant readTime)
+        String projectId,
+        String databaseId,
+        String ourKind,
+        @Nullable String namespace,
+        Datastore datastore,
+        @Nullable Instant readTime)
         throws DatastoreException {
-      long latestTimestamp = queryLatestStatisticsTimestamp(datastore, namespace, readTime);
+      long latestTimestamp =
+          queryLatestStatisticsTimestamp(datastore, projectId, databaseId, namespace, readTime);
       LOG.info("Latest stats timestamp for kind {} is {}", ourKind, latestTimestamp);
 
       Query.Builder queryBuilder = Query.newBuilder();
@@ -423,7 +450,8 @@ private static Entity getLatestTableStats(
               makeFilter("kind_name", EQUAL, makeValue(ourKind).build()).build(),
               makeFilter("timestamp", EQUAL, makeValue(latestTimestamp).build()).build()));
 
-      RunQueryRequest request = makeRequest(queryBuilder.build(), namespace, readTime);
+      RunQueryRequest request =
+          makeRequest(projectId, databaseId, queryBuilder.build(), namespace, readTime);
 
       long now = System.currentTimeMillis();
       RunQueryResponse response = datastore.runQuery(request);
@@ -447,10 +475,16 @@ private static Entity getLatestTableStats(
      * <p>See https://cloud.google.com/datastore/docs/concepts/stats.
      */
     static long getEstimatedSizeBytes(
-        Datastore datastore, Query query, @Nullable String namespace, @Nullable Instant readTime)
+        Datastore datastore,
+        String projectId,
+        String databaseId,
+        Query query,
+        @Nullable String namespace,
+        @Nullable Instant readTime)
         throws DatastoreException {
       String ourKind = query.getKind(0).getName();
-      Entity entity = getLatestTableStats(ourKind, namespace, datastore, readTime);
+      Entity entity =
+          getLatestTableStats(projectId, databaseId, ourKind, namespace, datastore, readTime);
       return entity.getPropertiesOrThrow("entity_bytes").getIntegerValue();
     }
 
@@ -470,9 +504,18 @@ private static PartitionId.Builder forNamespace(@Nullable String namespace) {
      * the requested {@code readTime}.
      */
     static RunQueryRequest makeRequest(
-        Query query, @Nullable String namespace, @Nullable Instant readTime) {
+        String projectId,
+        String databaseId,
+        Query query,
+        @Nullable String namespace,
+        @Nullable Instant readTime) {
       RunQueryRequest.Builder request =
-          RunQueryRequest.newBuilder().setQuery(query).setPartitionId(forNamespace(namespace));
+          RunQueryRequest.newBuilder()
+              .setProjectId(projectId)
+              .setDatabaseId(databaseId)
+              .setQuery(query)
+              .setPartitionId(
+                  forNamespace(namespace).setProjectId(projectId).setDatabaseId(databaseId));
       if (readTime != null) {
         Timestamp readTimeProto = Timestamps.fromMillis(readTime.getMillis());
         request.setReadOptions(ReadOptions.newBuilder().setReadTime(readTimeProto).build());
@@ -486,11 +529,18 @@ static RunQueryRequest makeRequest(
      * at the requested {@code readTime}.
      */
     static RunQueryRequest makeRequest(
-        GqlQuery gqlQuery, @Nullable String namespace, @Nullable Instant readTime) {
+        String projectId,
+        String databaseId,
+        GqlQuery gqlQuery,
+        @Nullable String namespace,
+        @Nullable Instant readTime) {
       RunQueryRequest.Builder request =
           RunQueryRequest.newBuilder()
+              .setProjectId(projectId)
+              .setDatabaseId(databaseId)
               .setGqlQuery(gqlQuery)
-              .setPartitionId(forNamespace(namespace));
+              .setPartitionId(
+                  forNamespace(namespace).setProjectId(projectId).setDatabaseId(databaseId));
       if (readTime != null) {
         Timestamp readTimeProto = Timestamps.fromMillis(readTime.getMillis());
         request.setReadOptions(ReadOptions.newBuilder().setReadTime(readTimeProto).build());
@@ -504,6 +554,8 @@ static RunQueryRequest makeRequest(
      * namespace}.
      */
     private static List<Query> splitQuery(
+        String projectId,
+        String databaseId,
         Query query,
         @Nullable String namespace,
         Datastore datastore,
@@ -512,7 +564,8 @@ private static List<Query> splitQuery(
         @Nullable Instant readTime)
         throws DatastoreException {
       // If namespace is set, include it in the split request so splits are calculated accordingly.
-      PartitionId partitionId = forNamespace(namespace).build();
+      PartitionId partitionId =
+          forNamespace(namespace).setProjectId(projectId).setDatabaseId(databaseId).build();
       if (readTime != null) {
         Timestamp readTimeProto = Timestamps.fromMillis(readTime.getMillis());
         return querySplitter.getSplits(query, partitionId, numSplits, datastore, readTimeProto);
@@ -535,12 +588,18 @@ private static List<Query> splitQuery(
      */
     @VisibleForTesting
     static Query translateGqlQueryWithLimitCheck(
-        String gql, Datastore datastore, String namespace, @Nullable Instant readTime)
+        String gql,
+        Datastore datastore,
+        String projectId,
+        String databaseId,
+        String namespace,
+        @Nullable Instant readTime)
         throws DatastoreException {
       String gqlQueryWithZeroLimit = gql + " LIMIT 0";
       try {
         Query translatedQuery =
-            translateGqlQuery(gqlQueryWithZeroLimit, datastore, namespace, readTime);
+            translateGqlQuery(
+                gqlQueryWithZeroLimit, datastore, projectId, databaseId, namespace, readTime);
         // Clear the limit that we set.
         return translatedQuery.toBuilder().clearLimit().build();
       } catch (DatastoreException e) {
@@ -551,7 +610,7 @@ static Query translateGqlQueryWithLimitCheck(
           LOG.warn("Failed to translate Gql query '{}': {}", gqlQueryWithZeroLimit, e.getMessage());
           LOG.warn("User query might have a limit already set, so trying without zero limit");
           // Retry without the zero limit.
-          return translateGqlQuery(gql, datastore, namespace, readTime);
+          return translateGqlQuery(gql, datastore, projectId, databaseId, namespace, readTime);
         } else {
           throw e;
         }
@@ -560,13 +619,27 @@ static Query translateGqlQueryWithLimitCheck(
 
     /** Translates a gql query string to {@link Query}. */
     private static Query translateGqlQuery(
-        String gql, Datastore datastore, String namespace, @Nullable Instant readTime)
+        String gql,
+        Datastore datastore,
+        String projectId,
+        String databaseId,
+        String namespace,
+        @Nullable Instant readTime)
         throws DatastoreException {
       GqlQuery gqlQuery = GqlQuery.newBuilder().setQueryString(gql).setAllowLiterals(true).build();
-      RunQueryRequest req = makeRequest(gqlQuery, namespace, readTime);
+      RunQueryRequest req = makeRequest(projectId, databaseId, gqlQuery, namespace, readTime);
       return datastore.runQuery(req).getQuery();
     }
 
+    /**
+     * Returns a new {@link DatastoreV1.Read} that reads from the Cloud Datastore for the specified
+     * database.
+     */
+    public DatastoreV1.Read withDatabaseId(String databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return toBuilder().setDatabaseId(StaticValueProvider.of(databaseId)).build();
+    }
+
     /**
      * Returns a new {@link DatastoreV1.Read} that reads from the Cloud Datastore for the specified
      * project.
@@ -677,13 +750,24 @@ public DatastoreV1.Read withReadTime(Instant readTime) {
     public long getNumEntities(
         PipelineOptions options, String ourKind, @Nullable String namespace) {
       try {
-        V1Options v1Options = V1Options.from(getProjectId(), getNamespace(), getLocalhost());
+        V1Options v1Options =
+            V1Options.from(getProjectId(), getDatabaseId(), getNamespace(), getLocalhost());
         V1DatastoreFactory datastoreFactory = new V1DatastoreFactory();
         Datastore datastore =
             datastoreFactory.getDatastore(
-                options, v1Options.getProjectId(), v1Options.getLocalhost());
-
-        Entity entity = getLatestTableStats(ourKind, namespace, datastore, getReadTime());
+                options,
+                v1Options.getProjectId(),
+                v1Options.getDatabaseId(),
+                v1Options.getLocalhost());
+
+        Entity entity =
+            getLatestTableStats(
+                v1Options.getProjectId(),
+                v1Options.getDatabaseId(),
+                ourKind,
+                namespace,
+                datastore,
+                getReadTime());
         return entity.getPropertiesOrThrow("count").getIntegerValue();
       } catch (Exception e) {
         return -1;
@@ -704,7 +788,8 @@ public PCollection<Entity> expand(PBegin input) {
           getQuery() == null || getLiteralGqlQuery() == null,
           "withQuery() and withLiteralGqlQuery() are exclusive");
 
-      V1Options v1Options = V1Options.from(getProjectId(), getNamespace(), getLocalhost());
+      V1Options v1Options =
+          V1Options.from(getProjectId(), getDatabaseId(), getNamespace(), getLocalhost());
 
       /*
        * This composite transform involves the following steps:
@@ -748,6 +833,7 @@ public void populateDisplayData(DisplayData.Builder builder) {
       String query = getQuery() == null ? null : getQuery().toString();
       builder
           .addIfNotNull(DisplayData.item("projectId", getProjectId()).withLabel("ProjectId"))
+          .addIfNotNull(DisplayData.item("databaseId", getDatabaseId()).withLabel("DatabaseId"))
           .addIfNotNull(DisplayData.item("namespace", getNamespace()).withLabel("Namespace"))
           .addIfNotNull(DisplayData.item("query", query).withLabel("Query"))
           .addIfNotNull(DisplayData.item("gqlQuery", getLiteralGqlQuery()).withLabel("GqlQuery"))
@@ -757,30 +843,46 @@ public void populateDisplayData(DisplayData.Builder builder) {
     @VisibleForTesting
     static class V1Options implements HasDisplayData, Serializable {
       private final ValueProvider<String> project;
+      private final ValueProvider<String> database;
       private final @Nullable ValueProvider<String> namespace;
       private final @Nullable String localhost;
 
       private V1Options(
-          ValueProvider<String> project, ValueProvider<String> namespace, String localhost) {
+          ValueProvider<String> project,
+          ValueProvider<String> database,
+          ValueProvider<String> namespace,
+          String localhost) {
         this.project = project;
+        this.database = database;
         this.namespace = namespace;
         this.localhost = localhost;
       }
 
-      public static V1Options from(String projectId, String namespace, String localhost) {
+      public static V1Options from(
+          String projectId, ValueProvider<String> databaseId, String namespace, String localhost) {
         return from(
-            StaticValueProvider.of(projectId), StaticValueProvider.of(namespace), localhost);
+            StaticValueProvider.of(projectId),
+            databaseId,
+            StaticValueProvider.of(namespace),
+            localhost);
       }
 
       public static V1Options from(
-          ValueProvider<String> project, ValueProvider<String> namespace, String localhost) {
-        return new V1Options(project, namespace, localhost);
+          ValueProvider<String> project,
+          ValueProvider<String> databaseId,
+          ValueProvider<String> namespace,
+          String localhost) {
+        return new V1Options(project, databaseId, namespace, localhost);
       }
 
       public String getProjectId() {
         return project.get();
       }
 
+      public String getDatabaseId() {
+        return database == null ? DEFAULT_DATABASE : database.get();
+      }
+
       public @Nullable String getNamespace() {
         return namespace == null ? null : namespace.get();
       }
@@ -789,6 +891,10 @@ public ValueProvider<String> getProjectValueProvider() {
         return project;
       }
 
+      public ValueProvider<String> getDatabaseValueProvider() {
+        return database;
+      }
+
       public @Nullable ValueProvider<String> getNamespaceValueProvider() {
         return namespace;
       }
@@ -802,6 +908,8 @@ public void populateDisplayData(DisplayData.Builder builder) {
         builder
             .addIfNotNull(
                 DisplayData.item("projectId", getProjectValueProvider()).withLabel("ProjectId"))
+            .addIfNotNull(
+                DisplayData.item("databaseId", getDatabaseValueProvider()).withLabel("DatabaseId"))
             .addIfNotNull(
                 DisplayData.item("namespace", getNamespaceValueProvider()).withLabel("Namespace"));
       }
@@ -833,7 +941,10 @@ static class GqlQueryTranslateFn extends DoFn<String, Query> {
       public void startBundle(StartBundleContext c) throws Exception {
         datastore =
             datastoreFactory.getDatastore(
-                c.getPipelineOptions(), v1Options.getProjectId(), v1Options.getLocalhost());
+                c.getPipelineOptions(),
+                v1Options.getProjectId(),
+                v1Options.getDatabaseId(),
+                v1Options.getLocalhost());
       }
 
       @ProcessElement
@@ -842,7 +953,12 @@ public void processElement(ProcessContext c) throws Exception {
         LOG.info("User query: '{}'", gqlQuery);
         Query query =
             translateGqlQueryWithLimitCheck(
-                gqlQuery, datastore, v1Options.getNamespace(), readTime);
+                gqlQuery,
+                datastore,
+                v1Options.getProjectId(),
+                v1Options.getDatabaseId(),
+                v1Options.getNamespace(),
+                readTime);
         LOG.info("User gql query translated to Query({})", query);
         c.output(query);
       }
@@ -890,7 +1006,10 @@ public SplitQueryFn(V1Options options, int numSplits, @Nullable Instant readTime
       public void startBundle(StartBundleContext c) throws Exception {
         datastore =
             datastoreFactory.getDatastore(
-                c.getPipelineOptions(), options.getProjectId(), options.getLocalhost());
+                c.getPipelineOptions(),
+                options.getProjectId(),
+                options.getDatabaseId(),
+                options.getLocalhost());
         querySplitter = datastoreFactory.getQuerySplitter();
       }
 
@@ -908,7 +1027,13 @@ public void processElement(ProcessContext c) throws Exception {
         // Compute the estimated numSplits if numSplits is not specified by the user.
         if (numSplits <= 0) {
           estimatedNumSplits =
-              getEstimatedNumSplits(datastore, query, options.getNamespace(), readTime);
+              getEstimatedNumSplits(
+                  datastore,
+                  options.getProjectId(),
+                  options.getDatabaseId(),
+                  query,
+                  options.getNamespace(),
+                  readTime);
         } else {
           estimatedNumSplits = numSplits;
         }
@@ -918,6 +1043,8 @@ public void processElement(ProcessContext c) throws Exception {
         try {
           querySplits =
               splitQuery(
+                  options.getProjectId(),
+                  options.getDatabaseId(),
                   query,
                   options.getNamespace(),
                   datastore,
@@ -985,7 +1112,10 @@ public ReadFn(V1Options options, @Nullable Instant readTime) {
       public void startBundle(StartBundleContext c) throws Exception {
         datastore =
             datastoreFactory.getDatastore(
-                c.getPipelineOptions(), options.getProjectId(), options.getLocalhost());
+                c.getPipelineOptions(),
+                options.getProjectId(),
+                options.getDatabaseId(),
+                options.getLocalhost());
       }
 
       private RunQueryResponse runQueryWithRetries(RunQueryRequest request) throws Exception {
@@ -1045,7 +1175,13 @@ public void processElement(ProcessContext context) throws Exception {
             queryBuilder.setStartCursor(currentBatch.getEndCursor());
           }
 
-          RunQueryRequest request = makeRequest(queryBuilder.build(), namespace, readTime);
+          RunQueryRequest request =
+              makeRequest(
+                  options.getProjectId(),
+                  options.getDatabaseId(),
+                  queryBuilder.build(),
+                  namespace,
+                  readTime);
           RunQueryResponse response = runQueryWithRetries(request);
 
           currentBatch = response.getBatch();
@@ -1128,21 +1264,42 @@ public static class Write extends Mutate<Entity> {
         @Nullable String localhost,
         boolean throttleRampup,
         ValueProvider<Integer> hintNumWorkers) {
-      super(projectId, localhost, new UpsertFn(), throttleRampup, hintNumWorkers);
+      super(projectId, null, localhost, new UpsertFn(), throttleRampup, hintNumWorkers);
+    }
+
+    Write(
+        @Nullable ValueProvider<String> projectId,
+        @Nullable ValueProvider<String> databaseId,
+        @Nullable String localhost,
+        boolean throttleRampup,
+        ValueProvider<Integer> hintNumWorkers) {
+      super(projectId, databaseId, localhost, new UpsertFn(), throttleRampup, hintNumWorkers);
     }
 
-    /** Returns a new {@link Write} that writes to the Cloud Datastore for the specified project. */
+    /** Returns a new {@link Write} that writes to the Cloud Datastore for the default database. */
     public Write withProjectId(String projectId) {
       checkArgument(projectId != null, "projectId can not be null");
       return withProjectId(StaticValueProvider.of(projectId));
     }
 
+    /** Returns a new {@link Write} that writes to the Cloud Datastore for the database id. */
+    public Write withDatabaseId(String databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return withDatabaseId(StaticValueProvider.of(databaseId));
+    }
+
     /** Same as {@link Write#withProjectId(String)} but with a {@link ValueProvider}. */
     public Write withProjectId(ValueProvider<String> projectId) {
       checkArgument(projectId != null, "projectId can not be null");
       return new Write(projectId, localhost, throttleRampup, hintNumWorkers);
     }
 
+    /** Same as {@link Write#withDatabaseId(String)} but with a {@link ValueProvider}. */
+    public Write withDatabaseId(ValueProvider<String> databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return new Write(projectId, databaseId, localhost, throttleRampup, hintNumWorkers);
+    }
+
     /**
      * Returns a new {@link Write} that writes to the Cloud Datastore Emulator running locally on
      * the specified host port.
@@ -1188,7 +1345,16 @@ public static class DeleteEntity extends Mutate<Entity> {
         @Nullable String localhost,
         boolean throttleRampup,
         ValueProvider<Integer> hintNumWorkers) {
-      super(projectId, localhost, new DeleteEntityFn(), throttleRampup, hintNumWorkers);
+      super(projectId, null, localhost, new DeleteEntityFn(), throttleRampup, hintNumWorkers);
+    }
+
+    DeleteEntity(
+        @Nullable ValueProvider<String> projectId,
+        @Nullable ValueProvider<String> databaseId,
+        @Nullable String localhost,
+        boolean throttleRampup,
+        ValueProvider<Integer> hintNumWorkers) {
+      super(projectId, databaseId, localhost, new DeleteEntityFn(), throttleRampup, hintNumWorkers);
     }
 
     /**
@@ -1200,12 +1366,27 @@ public DeleteEntity withProjectId(String projectId) {
       return withProjectId(StaticValueProvider.of(projectId));
     }
 
+    /**
+     * Returns a new {@link DeleteEntity} that deletes entities from the Cloud Datastore for the
+     * specified database.
+     */
+    public DeleteEntity withDatabaseId(String databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return withDatabaseId(StaticValueProvider.of(databaseId));
+    }
+
     /** Same as {@link DeleteEntity#withProjectId(String)} but with a {@link ValueProvider}. */
     public DeleteEntity withProjectId(ValueProvider<String> projectId) {
       checkArgument(projectId != null, "projectId can not be null");
       return new DeleteEntity(projectId, localhost, throttleRampup, hintNumWorkers);
     }
 
+    /** Same as {@link DeleteEntity#withDatabaseId(String)} but with a {@link ValueProvider}. */
+    public DeleteEntity withDatabaseId(ValueProvider<String> databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return new DeleteEntity(projectId, databaseId, localhost, throttleRampup, hintNumWorkers);
+    }
+
     /**
      * Returns a new {@link DeleteEntity} that deletes entities from the Cloud Datastore Emulator
      * running locally on the specified host port.
@@ -1253,7 +1434,16 @@ public static class DeleteKey extends Mutate<Key> {
         @Nullable String localhost,
         boolean throttleRampup,
         ValueProvider<Integer> hintNumWorkers) {
-      super(projectId, localhost, new DeleteKeyFn(), throttleRampup, hintNumWorkers);
+      super(projectId, null, localhost, new DeleteKeyFn(), throttleRampup, hintNumWorkers);
+    }
+
+    DeleteKey(
+        @Nullable ValueProvider<String> projectId,
+        @Nullable ValueProvider<String> databaseId,
+        @Nullable String localhost,
+        boolean throttleRampup,
+        ValueProvider<Integer> hintNumWorkers) {
+      super(projectId, databaseId, localhost, new DeleteKeyFn(), throttleRampup, hintNumWorkers);
     }
 
     /**
@@ -1265,6 +1455,15 @@ public DeleteKey withProjectId(String projectId) {
       return withProjectId(StaticValueProvider.of(projectId));
     }
 
+    /**
+     * Returns a new {@link DeleteKey} that deletes entities from the Cloud Datastore for the
+     * specified database.
+     */
+    public DeleteKey withDatabaseId(String databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return withDatabaseId(StaticValueProvider.of(databaseId));
+    }
+
     /**
      * Returns a new {@link DeleteKey} that deletes entities from the Cloud Datastore Emulator
      * running locally on the specified host port.
@@ -1280,6 +1479,12 @@ public DeleteKey withProjectId(ValueProvider<String> projectId) {
       return new DeleteKey(projectId, localhost, throttleRampup, hintNumWorkers);
     }
 
+    /** Same as {@link DeleteKey#withDatabaseId(String)} but with a {@link ValueProvider}. */
+    public DeleteKey withDatabaseId(ValueProvider<String> databaseId) {
+      checkArgument(databaseId != null, "databaseId can not be null");
+      return new DeleteKey(projectId, databaseId, localhost, throttleRampup, hintNumWorkers);
+    }
+
     /** Returns a new {@link DeleteKey} that does not throttle during ramp-up. */
     public DeleteKey withRampupThrottlingDisabled() {
       return new DeleteKey(projectId, localhost, false, hintNumWorkers);
@@ -1312,6 +1517,7 @@ public DeleteKey withHintNumWorkers(ValueProvider<Integer> hintNumWorkers) {
   private abstract static class Mutate<T> extends PTransform<PCollection<T>, PDone> {
 
     protected ValueProvider<String> projectId;
+    protected ValueProvider<String> databaseId;
     protected @Nullable String localhost;
     protected boolean throttleRampup;
     protected ValueProvider<Integer> hintNumWorkers;
@@ -1326,11 +1532,13 @@ private abstract static class Mutate<T> extends PTransform<PCollection<T>, PDone
      */
     Mutate(
         @Nullable ValueProvider<String> projectId,
+        @Nullable ValueProvider<String> databaseId,
         @Nullable String localhost,
         SimpleFunction<T, Mutation> mutationFn,
         boolean throttleRampup,
         ValueProvider<Integer> hintNumWorkers) {
       this.projectId = projectId;
+      this.databaseId = databaseId;
       this.localhost = localhost;
       this.throttleRampup = throttleRampup;
       this.hintNumWorkers = hintNumWorkers;
@@ -1372,7 +1580,14 @@ public PCollectionView<Instant> expand(PBegin input) {
                 ParDo.of(rampupThrottlingFn).withSideInputs(startTimestampView));
       }
       intermediateOutput.apply(
-          "Write Mutation to Datastore", ParDo.of(new DatastoreWriterFn(projectId, localhost)));
+          "Write Mutation to Datastore",
+          ParDo.of(
+              new DatastoreWriterFn(
+                  projectId,
+                  databaseId,
+                  localhost,
+                  new V1DatastoreFactory(),
+                  new WriteBatcherImpl())));
 
       return PDone.in(input.getPipeline());
     }
@@ -1390,6 +1605,7 @@ public void populateDisplayData(DisplayData.Builder builder) {
       super.populateDisplayData(builder);
       builder
           .addIfNotNull(DisplayData.item("projectId", projectId).withLabel("Output Project"))
+          .addIfNotNull(DisplayData.item("databaseId", databaseId).withLabel("Output Database"))
           .include("mutationFn", mutationFn);
       if (rampupThrottlingFn != null) {
         builder.include("rampupThrottlingFn", rampupThrottlingFn);
@@ -1399,6 +1615,10 @@ public void populateDisplayData(DisplayData.Builder builder) {
     public String getProjectId() {
       return projectId.get();
     }
+
+    public String getDatabaseId() {
+      return databaseId.get();
+    }
   }
 
   /** Determines batch sizes for commit RPCs. */
@@ -1480,6 +1700,7 @@ static class DatastoreWriterFn extends DoFn<Mutation, Void> {
 
     private static final Logger LOG = LoggerFactory.getLogger(DatastoreWriterFn.class);
     private final ValueProvider<String> projectId;
+    private final ValueProvider<String> databaseId;
     private final @Nullable String localhost;
     private transient Datastore datastore;
     private final V1DatastoreFactory datastoreFactory;
@@ -1511,13 +1732,14 @@ static class DatastoreWriterFn extends DoFn<Mutation, Void> {
     DatastoreWriterFn(String projectId, @Nullable String localhost) {
       this(
           StaticValueProvider.of(projectId),
+          null,
           localhost,
           new V1DatastoreFactory(),
           new WriteBatcherImpl());
     }
 
     DatastoreWriterFn(ValueProvider<String> projectId, @Nullable String localhost) {
-      this(projectId, localhost, new V1DatastoreFactory(), new WriteBatcherImpl());
+      this(projectId, null, localhost, new V1DatastoreFactory(), new WriteBatcherImpl());
     }
 
     @VisibleForTesting
@@ -1526,7 +1748,18 @@ static class DatastoreWriterFn extends DoFn<Mutation, Void> {
         @Nullable String localhost,
         V1DatastoreFactory datastoreFactory,
         WriteBatcher writeBatcher) {
+      this(projectId, null, localhost, datastoreFactory, writeBatcher);
+    }
+
+    @VisibleForTesting
+    DatastoreWriterFn(
+        ValueProvider<String> projectId,
+        ValueProvider<String> databaseId,
+        @Nullable String localhost,
+        V1DatastoreFactory datastoreFactory,
+        WriteBatcher writeBatcher) {
       this.projectId = checkNotNull(projectId, "projectId");
+      this.databaseId = databaseId;
       this.localhost = localhost;
       this.datastoreFactory = datastoreFactory;
       this.writeBatcher = writeBatcher;
@@ -1534,7 +1767,10 @@ static class DatastoreWriterFn extends DoFn<Mutation, Void> {
 
     @StartBundle
     public void startBundle(StartBundleContext c) {
-      datastore = datastoreFactory.getDatastore(c.getPipelineOptions(), projectId.get(), localhost);
+      String databaseIdOrDefaultDatabase = databaseId == null ? DEFAULT_DATABASE : databaseId.get();
+      datastore =
+          datastoreFactory.getDatastore(
+              c.getPipelineOptions(), projectId.get(), databaseIdOrDefaultDatabase, localhost);
       writeBatcher.start();
       if (adaptiveThrottler == null) {
         // Initialize throttler at first use, because it is not serializable.
@@ -1602,11 +1838,14 @@ private synchronized void flushBatch()
 
       batchSize.update(mutations.size());
 
+      String databaseIdOrDefaultDatabase = databaseId == null ? DEFAULT_DATABASE : databaseId.get();
       while (true) {
         // Batch upsert entities.
         CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
         commitRequest.addAllMutations(mutations);
         commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
+        commitRequest.setProjectId(projectId.get());
+        commitRequest.setDatabaseId(databaseIdOrDefaultDatabase);
         long startTime = System.currentTimeMillis(), endTime;
 
         if (adaptiveThrottler.throttleRequest(startTime)) {
@@ -1628,6 +1867,7 @@ private synchronized void flushBatch()
         ServiceCallMetric serviceCallMetric =
             new ServiceCallMetric(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, baseLabels);
         try {
+
           datastore.commit(commitRequest.build());
           endTime = System.currentTimeMillis();
           serviceCallMetric.call("ok");
@@ -1768,8 +2008,9 @@ public void populateDisplayData(DisplayData.Builder builder) {
   static class V1DatastoreFactory implements Serializable {
 
     /** Builds a Cloud Datastore client for the given pipeline options and project. */
-    public Datastore getDatastore(PipelineOptions pipelineOptions, String projectId) {
-      return getDatastore(pipelineOptions, projectId, null);
+    public Datastore getDatastore(
+        PipelineOptions pipelineOptions, String projectId, String databaseId) {
+      return getDatastore(pipelineOptions, projectId, databaseId, null);
     }
 
     /**
@@ -1777,7 +2018,10 @@ public Datastore getDatastore(PipelineOptions pipelineOptions, String projectId)
      * locahost.
      */
     public Datastore getDatastore(
-        PipelineOptions pipelineOptions, String projectId, @Nullable String localhost) {
+        PipelineOptions pipelineOptions,
+        String projectId,
+        String databaseId,
+        @Nullable String localhost) {
       Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
 
       // Add Beam version to user agent header.
@@ -1797,7 +2041,10 @@ public Datastore getDatastore(
       }
 
       DatastoreOptions.Builder builder =
-          new DatastoreOptions.Builder().projectId(projectId).initializer(initializer);
+          new DatastoreOptions.Builder()
+              .projectId(projectId)
+              .databaseId(databaseId)
+              .initializer(initializer);
 
       if (localhost != null) {
         builder.localHost(localhost);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
index f79299aea5fc..e281e559a544 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubIO.java
@@ -194,7 +194,7 @@ public class PubsubIO {
 
   private static final Pattern PUBSUB_NAME_REGEXP = Pattern.compile("[a-zA-Z][-._~%+a-zA-Z0-9]+");
 
-  static final int PUBSUB_MESSAGE_MAX_TOTAL_SIZE = 10 << 20;
+  static final int PUBSUB_MESSAGE_MAX_TOTAL_SIZE = 10_000_000;
 
   private static final int PUBSUB_NAME_MIN_LENGTH = 3;
   private static final int PUBSUB_NAME_MAX_LENGTH = 255;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java
index f6acd081ab23..98c5f8a6b388 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteReadSchemaTransformProvider.java
@@ -23,14 +23,22 @@
 import com.google.cloud.pubsublite.ProjectId;
 import com.google.cloud.pubsublite.SubscriptionName;
 import com.google.cloud.pubsublite.SubscriptionPath;
+import com.google.cloud.pubsublite.proto.AttributeValues;
+import com.google.cloud.pubsublite.proto.PubSubMessage;
 import com.google.cloud.pubsublite.proto.SequencedMessage;
+import com.google.protobuf.ByteString;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Consumer;
 import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.gcp.pubsublite.internal.Uuid;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.schemas.AutoValueSchema;
@@ -40,12 +48,13 @@
 import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
 import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider;
 import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider;
+import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling;
 import org.apache.beam.sdk.schemas.utils.JsonUtils;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.FinishBundle;
-import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionRowTuple;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.Row;
@@ -68,14 +77,12 @@ public class PubsubLiteReadSchemaTransformProvider
   private static final Logger LOG =
       LoggerFactory.getLogger(PubsubLiteReadSchemaTransformProvider.class);
 
-  public static final String VALID_FORMATS_STR = "AVRO,JSON";
+  public static final String VALID_FORMATS_STR = "RAW,AVRO,JSON";
   public static final Set<String> VALID_DATA_FORMATS =
       Sets.newHashSet(VALID_FORMATS_STR.split(","));
 
   public static final TupleTag<Row> OUTPUT_TAG = new TupleTag<Row>() {};
   public static final TupleTag<Row> ERROR_TAG = new TupleTag<Row>() {};
-  public static final Schema ERROR_SCHEMA =
-      Schema.builder().addStringField("error").addNullableByteArrayField("row").build();
 
   @Override
   protected @UnknownKeyFor @NonNull @Initialized Class<PubsubLiteReadSchemaTransformConfiguration>
@@ -84,30 +91,95 @@ public class PubsubLiteReadSchemaTransformProvider
   }
 
   public static class ErrorFn extends DoFn<SequencedMessage, Row> {
-    private SerializableFunction<byte[], Row> valueMapper;
-    private Counter errorCounter;
+    private final SerializableFunction<byte[], Row> valueMapper;
+    private final Counter errorCounter;
     private Long errorsInBundle = 0L;
+    private final boolean handleErrors;
 
-    public ErrorFn(String name, SerializableFunction<byte[], Row> valueMapper) {
+    private final List<String> attributes;
+
+    private final String attributeMap;
+
+    private final Schema errorSchema;
+
+    private final Schema attributeSchema;
+
+    public ErrorFn(
+        String name,
+        SerializableFunction<byte[], Row> valueMapper,
+        Schema errorSchema,
+        boolean handleErrors) {
+      this.errorCounter = Metrics.counter(PubsubLiteReadSchemaTransformProvider.class, name);
+      this.valueMapper = valueMapper;
+      this.errorSchema = errorSchema;
+      this.handleErrors = handleErrors;
+      this.attributes = new ArrayList<>();
+      this.attributeMap = "";
+      this.attributeSchema = Schema.builder().build();
+    }
+
+    public ErrorFn(
+        String name,
+        SerializableFunction<byte[], Row> valueMapper,
+        Schema errorSchema,
+        List<String> attributes,
+        String attributeMap,
+        Schema attributeSchema,
+        boolean handleErrors) {
       this.errorCounter = Metrics.counter(PubsubLiteReadSchemaTransformProvider.class, name);
       this.valueMapper = valueMapper;
+      this.errorSchema = errorSchema;
+      this.handleErrors = handleErrors;
+      this.attributes = attributes;
+      this.attributeMap = attributeMap;
+      this.attributeSchema = attributeSchema;
     }
 
     @ProcessElement
     public void process(@DoFn.Element SequencedMessage seqMessage, MultiOutputReceiver receiver) {
+      Row mappedRow = null;
       try {
-        receiver
-            .get(OUTPUT_TAG)
-            .output(valueMapper.apply(seqMessage.getMessage().getData().toByteArray()));
+        if (attributes.isEmpty()
+            && attributeSchema.getFields().isEmpty()
+            && attributeMap.isEmpty()) {
+          mappedRow = valueMapper.apply(seqMessage.getMessage().getData().toByteArray());
+        } else {
+          PubSubMessage message = seqMessage.getMessage();
+          Row row = valueMapper.apply(message.getData().toByteArray());
+          Row.Builder rowBuilder = Row.withSchema(attributeSchema).addValues(row.getValues());
+          Map<String, String> stringAttributeMap = new HashMap<>();
+          message
+              .getAttributesMap()
+              .forEach(
+                  (attributeName, attributeValues) -> {
+                    if (attributes.contains(attributeName)) {
+                      processAttribute(attributeValues, rowBuilder::addValue);
+                    }
+
+                    if (!attributeMap.isEmpty()) {
+                      processAttribute(
+                          attributeValues, value -> stringAttributeMap.put(attributeName, value));
+                    }
+                  });
+          if (!attributeMap.isEmpty() && !stringAttributeMap.isEmpty()) {
+            rowBuilder.addValue(stringAttributeMap);
+          }
+          mappedRow = rowBuilder.build();
+        }
       } catch (Exception e) {
+        if (!handleErrors) {
+          throw new RuntimeException(e);
+        }
         errorsInBundle += 1;
         LOG.warn("Error while parsing the element", e);
         receiver
             .get(ERROR_TAG)
             .output(
-                Row.withSchema(ERROR_SCHEMA)
-                    .addValues(e.toString(), seqMessage.getMessage().getData().toByteArray())
-                    .build());
+                ErrorHandling.errorRecord(
+                    errorSchema, seqMessage.getMessage().getData().toByteArray(), e));
+      }
+      if (mappedRow != null) {
+        receiver.get(OUTPUT_TAG).output(mappedRow);
       }
     }
 
@@ -127,15 +199,35 @@ public void finish(FinishBundleContext c) {
               "Format %s not supported. Only supported formats are %s",
               configuration.getFormat(), VALID_FORMATS_STR));
     }
-    final Schema beamSchema =
-        Objects.equals(configuration.getFormat(), "JSON")
-            ? JsonUtils.beamSchemaFromJsonSchema(configuration.getSchema())
-            : AvroUtils.toBeamSchema(
-                new org.apache.avro.Schema.Parser().parse(configuration.getSchema()));
-    final SerializableFunction<byte[], Row> valueMapper =
-        Objects.equals(configuration.getFormat(), "JSON")
-            ? JsonUtils.getJsonBytesToRowFunction(beamSchema)
-            : AvroUtils.getAvroBytesToRowFunction(beamSchema);
+    boolean handleErrors = ErrorHandling.hasOutput(configuration.getErrorHandling());
+    String format = configuration.getFormat();
+    String inputSchema = configuration.getSchema();
+    List<String> attributes = configuration.getAttributes();
+    SerializableFunction<byte[], Row> valueMapper;
+    Schema beamSchema;
+
+    if (format != null && format.equals("RAW")) {
+      if (inputSchema != null) {
+        throw new IllegalArgumentException(
+            "To read from PubSubLite in RAW format, you can't provide a schema.");
+      }
+      beamSchema = Schema.builder().addField("payload", Schema.FieldType.BYTES).build();
+      valueMapper = getRawBytesToRowFunction(beamSchema);
+
+    } else {
+      if (inputSchema == null) {
+        throw new IllegalArgumentException(
+            "To read from PubSubLite in JSON or AVRO format, you must provide a schema.");
+      }
+      beamSchema =
+          Objects.equals(configuration.getFormat(), "JSON")
+              ? JsonUtils.beamSchemaFromJsonSchema(inputSchema)
+              : AvroUtils.toBeamSchema(new org.apache.avro.Schema.Parser().parse(inputSchema));
+      valueMapper =
+          Objects.equals(configuration.getFormat(), "JSON")
+              ? JsonUtils.getJsonBytesToRowFunction(beamSchema)
+              : AvroUtils.getAvroBytesToRowFunction(beamSchema);
+    }
     return new SchemaTransform() {
       @Override
       public PCollectionRowTuple expand(PCollectionRowTuple input) {
@@ -147,7 +239,16 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
           throw new IllegalArgumentException(
               "Unable to infer the project to read from Pubsub Lite. Please provide a project.");
         }
-        PCollectionTuple outputTuple =
+        Schema errorSchema = ErrorHandling.errorSchemaBytes();
+        List<String> attributeList = new ArrayList<>();
+        if (attributes != null) {
+          attributeList = attributes;
+        }
+        String attributeMapValue = configuration.getAttributeMap();
+        String attributeMap = attributeMapValue == null ? "" : attributeMapValue;
+        Schema resultingBeamSchema =
+            buildSchemaWithAttributes(beamSchema, attributeList, attributeMap);
+        PCollection<SequencedMessage> readPubsubLite =
             input
                 .getPipeline()
                 .apply(
@@ -161,16 +262,124 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
                                     .setName(
                                         SubscriptionName.of(configuration.getSubscriptionName()))
                                     .build())
-                            .build()))
-                .apply(
-                    ParDo.of(new ErrorFn("PubsubLite-read-error-counter", valueMapper))
-                        .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+                            .build()));
 
+        String attributeId = configuration.getAttributeId();
+        PCollectionTuple outputTuple;
+        PCollection<SequencedMessage> transformSequencedMessage;
+        if (attributeId != null && !attributeId.isEmpty()) {
+          UuidDeduplicationOptions.Builder uuidExtractor =
+              UuidDeduplicationOptions.newBuilder()
+                  .setUuidExtractor(getUuidFromMessage(attributeId));
+          transformSequencedMessage =
+              readPubsubLite.apply(PubsubLiteIO.deduplicate(uuidExtractor.build()));
+        } else {
+          transformSequencedMessage = readPubsubLite;
+        }
+
+        outputTuple =
+            transformSequencedMessage.apply(
+                ParDo.of(
+                        new ErrorFn(
+                            "PubsubLite-read-error-counter",
+                            valueMapper,
+                            errorSchema,
+                            attributeList,
+                            attributeMap,
+                            resultingBeamSchema,
+                            handleErrors))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
         return PCollectionRowTuple.of(
             "output",
-            outputTuple.get(OUTPUT_TAG).setRowSchema(beamSchema),
+            outputTuple.get(OUTPUT_TAG).setRowSchema(resultingBeamSchema),
             "errors",
-            outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA));
+            outputTuple.get(ERROR_TAG).setRowSchema(errorSchema));
+      }
+    };
+  }
+
+  /**
+   * Builds a new {@link Schema} by adding additional optional attributes and map field to the
+   * provided schema.
+   *
+   * @param schema The base schema to which additional attributes and map field will be added.
+   * @param attributes A list of optional attribute names to be added as STRING fields to the
+   *     schema.
+   * @param attributesMap The name of the optional map field to be added to the schema. If empty, no
+   *     map field will be added.
+   * @return A new {@link Schema} with the specified attributes and an optional map field.
+   * @throws IllegalArgumentException if the schema is null or if any attribute name in the
+   *     attributes list is null or empty.
+   */
+  public static Schema buildSchemaWithAttributes(
+      Schema schema, List<String> attributes, String attributesMap) {
+    Schema.Builder schemaBuilder = Schema.builder();
+    // Copy fields from the original schema
+    schema.getFields().forEach(field -> schemaBuilder.addField(field.getName(), field.getType()));
+
+    // Add optional additional attributes as STRING fields
+    attributes.forEach(
+        attribute -> {
+          if (attribute == null || attribute.isEmpty()) {
+            throw new IllegalArgumentException(
+                "Attribute names in the attributes list must not be null or empty.");
+          }
+          schemaBuilder.addField(attribute, Schema.FieldType.STRING);
+        });
+
+    // Add an optional map field if attributesMap is not empty
+    if (!attributesMap.isEmpty()) {
+      schemaBuilder
+          .addMapField(attributesMap, Schema.FieldType.STRING, Schema.FieldType.STRING)
+          .build();
+    }
+    return schemaBuilder.build();
+  }
+
+  /**
+   * Processes the attribute values, invoking the specified consumer with the processed value. If
+   * the attribute values are null or contain multiple values, an exception is thrown.
+   *
+   * @param attributeValues The attribute values to be processed. If null, the method does nothing.
+   * @param valueConsumer The consumer to accept the processed value.
+   * @throws RuntimeException if attributeValues is not null and contains multiple values.
+   */
+  private static void processAttribute(
+      @Nullable AttributeValues attributeValues, Consumer<String> valueConsumer) {
+    if (attributeValues != null) {
+      List<ByteString> valueList = attributeValues.getValuesList();
+      if (valueList.size() != 1) {
+        throw new RuntimeException(
+            "Received an unparseable message with multiple values for an attribute.");
+      }
+      valueConsumer.accept(valueList.get(0).toStringUtf8());
+    }
+  }
+
+  public static SerializableFunction<byte[], Row> getRawBytesToRowFunction(Schema rawSchema) {
+    return new SimpleFunction<byte[], Row>() {
+      @Override
+      public Row apply(byte[] input) {
+        return Row.withSchema(rawSchema).addValue(input).build();
+      }
+    };
+  }
+
+  public static SerializableFunction<SequencedMessage, Uuid> getUuidFromMessage(
+      String attributeId) {
+    return new SimpleFunction<SequencedMessage, Uuid>() {
+      @Override
+      public Uuid apply(SequencedMessage input) {
+        AttributeValues attribute = input.getMessage().getAttributesMap().get(attributeId);
+        if (attribute != null) {
+          if (attribute.getValuesCount() != 1) {
+            throw new RuntimeException(
+                "Received an unparseable message with multiple values for an attribute.");
+          }
+          return Uuid.of(attribute.getValues(0));
+        } else {
+          throw new RuntimeException("Uuid attribute missing.");
+        }
       }
     };
   }
@@ -205,7 +414,7 @@ public abstract static class PubsubLiteReadSchemaTransformConfiguration {
             + "For AVRO data, this is a schema defined with AVRO schema syntax "
             + "(https://avro.apache.org/docs/1.10.2/spec.html#schemas). "
             + "For JSON data, this is a schema defined with JSON-schema syntax (https://json-schema.org/).")
-    public abstract String getSchema();
+    public abstract @Nullable String getSchema();
 
     @SchemaFieldDescription(
         "The GCP project where the Pubsub Lite reservation resides. This can be a "
@@ -220,6 +429,36 @@ public abstract static class PubsubLiteReadSchemaTransformConfiguration {
     @SchemaFieldDescription("The region or zone where the Pubsub Lite reservation resides.")
     public abstract String getLocation();
 
+    @SchemaFieldDescription("This option specifies whether and where to output unwritable rows.")
+    public abstract @Nullable ErrorHandling getErrorHandling();
+
+    @SchemaFieldDescription(
+        "List of attribute keys whose values will be flattened into the "
+            + "output message as additional fields.  For example, if the format is `RAW` "
+            + "and attributes is `[\"a\", \"b\"]` then this read will produce elements of "
+            + "the form `Row(payload=..., a=..., b=...)`")
+    public abstract @Nullable List<String> getAttributes();
+
+    @SchemaFieldDescription(
+        "Name of a field in which to store the full set of attributes "
+            + "associated with this message.  For example, if the format is `RAW` and "
+            + "`attribute_map` is set to `\"attrs\"` then this read will produce elements "
+            + "of the form `Row(payload=..., attrs=...)` where `attrs` is a Map type "
+            + "of string to string. "
+            + "If both `attributes` and `attribute_map` are set, the overlapping "
+            + "attribute values will be present in both the flattened structure and the "
+            + "attribute map.")
+    public abstract @Nullable String getAttributeMap();
+
+    @SchemaFieldDescription(
+        "The attribute on incoming Pubsub Lite messages to use as a unique "
+            + "record identifier. When specified, the value of this attribute (which "
+            + "can be any string that uniquely identifies the record) will be used for "
+            + "deduplication of messages. If not provided, we cannot guarantee "
+            + "that no duplicate data will be delivered on the Pub/Sub stream. In this "
+            + "case, deduplication of the stream will be strictly best effort.")
+    public abstract @Nullable String getAttributeId();
+
     public static Builder builder() {
       return new AutoValue_PubsubLiteReadSchemaTransformProvider_PubsubLiteReadSchemaTransformConfiguration
           .Builder();
@@ -237,6 +476,16 @@ public abstract static class Builder {
 
       public abstract Builder setLocation(String location);
 
+      public abstract Builder setErrorHandling(ErrorHandling errorHandling);
+
+      public abstract Builder setAttributes(List<String> attributes);
+
+      @SuppressWarnings("unused")
+      public abstract Builder setAttributeMap(String attributeMap);
+
+      @SuppressWarnings("unused")
+      public abstract Builder setAttributeId(String attributeId);
+
       public abstract PubsubLiteReadSchemaTransformConfiguration build();
     }
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java
index 1b4d2bd77fb5..c669f84f330f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteWriteSchemaTransformProvider.java
@@ -23,14 +23,18 @@
 import com.google.cloud.pubsublite.ProjectId;
 import com.google.cloud.pubsublite.TopicName;
 import com.google.cloud.pubsublite.TopicPath;
+import com.google.cloud.pubsublite.proto.AttributeValues;
 import com.google.cloud.pubsublite.proto.PubSubMessage;
 import com.google.protobuf.ByteString;
-import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils;
+import org.apache.beam.sdk.io.gcp.pubsublite.internal.Uuid;
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.schemas.AutoValueSchema;
@@ -40,11 +44,15 @@
 import org.apache.beam.sdk.schemas.transforms.SchemaTransform;
 import org.apache.beam.sdk.schemas.transforms.SchemaTransformProvider;
 import org.apache.beam.sdk.schemas.transforms.TypedSchemaTransformProvider;
+import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling;
 import org.apache.beam.sdk.schemas.utils.JsonUtils;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.ProcessElement;
+import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionRowTuple;
 import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.Row;
@@ -53,6 +61,7 @@
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets;
 import org.checkerframework.checker.initialization.qual.Initialized;
 import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
 import org.checkerframework.checker.nullness.qual.UnknownKeyFor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,13 +71,11 @@ public class PubsubLiteWriteSchemaTransformProvider
     extends TypedSchemaTransformProvider<
         PubsubLiteWriteSchemaTransformProvider.PubsubLiteWriteSchemaTransformConfiguration> {
 
-  public static final String SUPPORTED_FORMATS_STR = "JSON,AVRO";
+  public static final String SUPPORTED_FORMATS_STR = "RAW,JSON,AVRO";
   public static final Set<String> SUPPORTED_FORMATS =
       Sets.newHashSet(SUPPORTED_FORMATS_STR.split(","));
   public static final TupleTag<PubSubMessage> OUTPUT_TAG = new TupleTag<PubSubMessage>() {};
   public static final TupleTag<Row> ERROR_TAG = new TupleTag<Row>() {};
-  public static final Schema ERROR_SCHEMA =
-      Schema.builder().addStringField("error").addNullableByteArrayField("row").build();
   private static final Logger LOG =
       LoggerFactory.getLogger(PubsubLiteWriteSchemaTransformProvider.class);
 
@@ -79,33 +86,80 @@ public class PubsubLiteWriteSchemaTransformProvider
   }
 
   public static class ErrorCounterFn extends DoFn<Row, PubSubMessage> {
-    private SerializableFunction<Row, byte[]> toBytesFn;
-    private Counter errorCounter;
+    private final SerializableFunction<Row, byte[]> toBytesFn;
+    private final Counter errorCounter;
     private long errorsInBundle = 0L;
 
-    public ErrorCounterFn(String name, SerializableFunction<Row, byte[]> toBytesFn) {
+    private final Schema errorSchema;
+
+    private final boolean handleErrors;
+
+    private final List<String> attributes;
+
+    private final Schema schemaWithoutAttributes;
+
+    public ErrorCounterFn(
+        String name,
+        SerializableFunction<Row, byte[]> toBytesFn,
+        Schema errorSchema,
+        boolean handleErrors) {
       this.toBytesFn = toBytesFn;
       errorCounter = Metrics.counter(PubsubLiteWriteSchemaTransformProvider.class, name);
+      this.errorSchema = errorSchema;
+      this.handleErrors = handleErrors;
+      this.attributes = new ArrayList<>();
+      this.schemaWithoutAttributes = Schema.builder().build();
+    }
+
+    public ErrorCounterFn(
+        String name,
+        SerializableFunction<Row, byte[]> toBytesFn,
+        Schema errorSchema,
+        boolean handleErrors,
+        List<String> attributes,
+        Schema schemaWithoutAttributes) {
+      this.toBytesFn = toBytesFn;
+      errorCounter = Metrics.counter(PubsubLiteWriteSchemaTransformProvider.class, name);
+      this.errorSchema = errorSchema;
+      this.handleErrors = handleErrors;
+      this.attributes = attributes;
+      this.schemaWithoutAttributes = schemaWithoutAttributes;
     }
 
     @ProcessElement
     public void process(@DoFn.Element Row row, MultiOutputReceiver receiver) {
       try {
-        PubSubMessage message =
-            PubSubMessage.newBuilder()
-                .setData(ByteString.copyFrom(Objects.requireNonNull(toBytesFn.apply(row))))
-                .build();
+        PubSubMessage message;
+        if (attributes.isEmpty()) {
+          message =
+              PubSubMessage.newBuilder()
+                  .setData(ByteString.copyFrom(Objects.requireNonNull(toBytesFn.apply(row))))
+                  .build();
+        } else {
+          Row.Builder builder = Row.withSchema(schemaWithoutAttributes);
+          schemaWithoutAttributes
+              .getFields()
+              .forEach(field -> builder.addValue(row.getValue(field.getName())));
+
+          Row resultingRow = builder.build();
+          Map<String, AttributeValues> attributeValuesHashMap =
+              getStringAttributeValuesMap(row, attributes);
+          message =
+              PubSubMessage.newBuilder()
+                  .setData(
+                      ByteString.copyFrom(Objects.requireNonNull(toBytesFn.apply(resultingRow))))
+                  .putAllAttributes(attributeValuesHashMap)
+                  .build();
+        }
 
         receiver.get(OUTPUT_TAG).output(message);
       } catch (Exception e) {
+        if (!handleErrors) {
+          throw new RuntimeException(e);
+        }
         errorsInBundle += 1;
-        LOG.warn("Error while parsing the element", e);
-        receiver
-            .get(ERROR_TAG)
-            .output(
-                Row.withSchema(ERROR_SCHEMA)
-                    .addValues(e.toString(), row.toString().getBytes(StandardCharsets.UTF_8))
-                    .build());
+        LOG.warn("Error while processing the element", e);
+        receiver.get(ERROR_TAG).output(ErrorHandling.errorRecord(errorSchema, row, e));
       }
     }
 
@@ -132,23 +186,59 @@ public void finish() {
     return new SchemaTransform() {
       @Override
       public PCollectionRowTuple expand(PCollectionRowTuple input) {
-        Schema inputSchema = input.get("input").getSchema();
-        final SerializableFunction<Row, byte[]> toBytesFn =
-            configuration.getFormat().equals("JSON")
-                ? JsonUtils.getRowToJsonBytesFunction(inputSchema)
-                : AvroUtils.getRowToAvroBytesFunction(inputSchema);
+        List<String> attributesConfigValue = configuration.getAttributes();
+        String attributeId = configuration.getAttributeId();
+        List<String> attributes =
+            attributesConfigValue != null ? attributesConfigValue : new ArrayList<>();
+        Schema inputSchema;
+        if (!attributes.isEmpty()) {
+          inputSchema = getSchemaWithoutAttributes(input.get("input").getSchema(), attributes);
+        } else {
+          inputSchema = input.get("input").getSchema();
+        }
+        ErrorHandling errorHandling = configuration.getErrorHandling();
+        boolean handleErrors = ErrorHandling.hasOutput(errorHandling);
+        Schema errorSchema = ErrorHandling.errorSchema(inputSchema);
+
+        final SerializableFunction<Row, byte[]> toBytesFn;
+        if (configuration.getFormat().equals("RAW")) {
+          int numFields = inputSchema.getFields().size();
+          if (numFields != 1) {
+            throw new IllegalArgumentException("Expecting exactly one field, found " + numFields);
+          }
+          if (!inputSchema.getField(0).getType().equals(Schema.FieldType.BYTES)) {
+            throw new IllegalArgumentException(
+                "The input schema must have exactly one field of type byte.");
+          }
+          toBytesFn = getRowToRawBytesFunction(inputSchema.getField(0).getName());
+        } else if (configuration.getFormat().equals("JSON")) {
+          toBytesFn = JsonUtils.getRowToJsonBytesFunction(inputSchema);
+        } else {
+          toBytesFn = AvroUtils.getRowToAvroBytesFunction(inputSchema);
+        }
 
         PCollectionTuple outputTuple =
             input
                 .get("input")
                 .apply(
                     "Map Rows to PubSubMessages",
-                    ParDo.of(new ErrorCounterFn("PubSubLite-write-error-counter", toBytesFn))
+                    ParDo.of(
+                            new ErrorCounterFn(
+                                "PubSubLite-write-error-counter",
+                                toBytesFn,
+                                errorSchema,
+                                handleErrors,
+                                attributes,
+                                inputSchema))
                         .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
 
         outputTuple
             .get(OUTPUT_TAG)
-            .apply("Add UUIDs", PubsubLiteIO.addUuids())
+            .apply(
+                "Add UUIDs",
+                (attributeId != null && !attributeId.isEmpty())
+                    ? new SetUuidFromPubSubMessage(attributeId)
+                    : PubsubLiteIO.addUuids())
             .apply(
                 "Write to PS Lite",
                 PubsubLiteIO.write(
@@ -161,8 +251,53 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
                                 .build())
                         .build()));
 
-        return PCollectionRowTuple.of(
-            "errors", outputTuple.get(ERROR_TAG).setRowSchema(ERROR_SCHEMA));
+        PCollection<Row> errorOutput =
+            outputTuple.get(ERROR_TAG).setRowSchema(ErrorHandling.errorSchema(errorSchema));
+
+        String outputString = errorHandling != null ? errorHandling.getOutput() : "errors";
+        return PCollectionRowTuple.of(handleErrors ? outputString : "errors", errorOutput);
+      }
+    };
+  }
+
+  public static Schema getSchemaWithoutAttributes(Schema inputSchema, List<String> attributes) {
+    Schema.Builder schemaBuilder = Schema.builder();
+
+    inputSchema
+        .getFields()
+        .forEach(
+            field -> {
+              if (!attributes.contains(field.getName())) {
+                schemaBuilder.addField(field.getName(), field.getType());
+              }
+            });
+    return schemaBuilder.build();
+  }
+
+  private static Map<String, AttributeValues> getStringAttributeValuesMap(
+      Row row, List<String> attributes) {
+    Map<String, AttributeValues> attributeValuesHashMap = new HashMap<>();
+    attributes.forEach(
+        attribute -> {
+          String value = row.getValue(attribute);
+          if (value != null) {
+            attributeValuesHashMap.put(
+                attribute,
+                AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8(value)).build());
+          }
+        });
+    return attributeValuesHashMap;
+  }
+
+  public static SerializableFunction<Row, byte[]> getRowToRawBytesFunction(String rowFieldName) {
+    return new SimpleFunction<Row, byte[]>() {
+      @Override
+      public byte[] apply(Row input) {
+        byte[] rawBytes = input.getBytes(rowFieldName);
+        if (rawBytes == null) {
+          throw new NullPointerException();
+        }
+        return rawBytes;
       }
     };
   }
@@ -205,6 +340,24 @@ public abstract static class PubsubLiteWriteSchemaTransformConfiguration {
             + SUPPORTED_FORMATS_STR)
     public abstract String getFormat();
 
+    @SchemaFieldDescription("This option specifies whether and where to output unwritable rows.")
+    public abstract @Nullable ErrorHandling getErrorHandling();
+
+    @SchemaFieldDescription(
+        "List of attribute keys whose values will be pulled out as "
+            + "Pubsub Lite message attributes.  For example, if the format is `JSON` "
+            + "and attributes is `[\"a\", \"b\"]` then elements of the form "
+            + "`Row(any_field=..., a=..., b=...)` will result in Pubsub Lite messages whose "
+            + "payload has the contents of any_field and whose attribute will be "
+            + "populated with the values of `a` and `b`.")
+    public abstract @Nullable List<String> getAttributes();
+
+    @SchemaFieldDescription(
+        "If set, will set an attribute for each Pubsub Lite message "
+            + "with the given name and a unique value. This attribute can then be used "
+            + "in a ReadFromPubSubLite PTransform to deduplicate messages.")
+    public abstract @Nullable String getAttributeId();
+
     public static Builder builder() {
       return new AutoValue_PubsubLiteWriteSchemaTransformProvider_PubsubLiteWriteSchemaTransformConfiguration
           .Builder();
@@ -220,7 +373,45 @@ public abstract static class Builder {
 
       public abstract Builder setFormat(String format);
 
+      public abstract Builder setErrorHandling(ErrorHandling errorHandling);
+
+      public abstract Builder setAttributes(List<String> attributes);
+
+      @SuppressWarnings("unused")
+      public abstract Builder setAttributeId(String attributeId);
+
       public abstract PubsubLiteWriteSchemaTransformConfiguration build();
     }
   }
+
+  public static class SetUuidFromPubSubMessage
+      extends PTransform<PCollection<PubSubMessage>, PCollection<PubSubMessage>> {
+    private final String attributeId;
+
+    public SetUuidFromPubSubMessage(String attributeId) {
+      this.attributeId = attributeId;
+    }
+
+    @Override
+    public PCollection<PubSubMessage> expand(PCollection<PubSubMessage> input) {
+      return input.apply("SetUuidFromPubSubMessage", ParDo.of(new SetUuidFn(attributeId)));
+    }
+
+    public static class SetUuidFn extends DoFn<PubSubMessage, PubSubMessage> {
+      private final String attributeId;
+
+      public SetUuidFn(String attributeId) {
+        this.attributeId = attributeId;
+      }
+
+      @ProcessElement
+      public void processElement(
+          @Element PubSubMessage input, OutputReceiver<PubSubMessage> outputReceiver) {
+        PubSubMessage.Builder builder = input.toBuilder();
+        builder.putAttributes(
+            attributeId, AttributeValues.newBuilder().addValues(Uuid.random().value()).build());
+        outputReceiver.output(builder.build());
+      }
+    }
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java
index 5e5c215fdebc..1e25d1e72abb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/BatchSpannerRead.java
@@ -246,7 +246,8 @@ public void processElement(ProcessContext c) throws Exception {
         }
       } catch (SpannerException e) {
         serviceCallMetric.call(e.getErrorCode().getGrpcStatusCode().toString());
-        LOG.error("Error while processing element", e);
+        LOG.error(
+            "Error while reading partition for operation: " + op.getReadOperation().toString(), e);
         throw (e);
       }
       serviceCallMetric.call("ok");
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java
index 45cc331657f1..2cc07b888983 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/NaiveSpannerRead.java
@@ -34,6 +34,8 @@
 import org.apache.beam.sdk.values.PCollectionView;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
 import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** A naive version of Spanner read that doesn't use the Batch API. */
 @VisibleForTesting
@@ -44,6 +46,8 @@
 abstract class NaiveSpannerRead
     extends PTransform<PCollection<ReadOperation>, PCollection<Struct>> {
 
+  private static final Logger LOG = LoggerFactory.getLogger(NaiveSpannerRead.class);
+
   public static NaiveSpannerRead create(
       SpannerConfig spannerConfig,
       PCollectionView<Transaction> txView,
@@ -109,6 +113,7 @@ public void processElement(ProcessContext c) throws Exception {
         }
       } catch (SpannerException e) {
         serviceCallMetric.call(e.getErrorCode().getGrpcStatusCode().toString());
+        LOG.error("Error while reading operation: " + op.toString(), e);
         throw (e);
       }
       serviceCallMetric.call("ok");
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java
index 959b0cb55025..4cbfe12b231f 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/ReadOperation.java
@@ -69,6 +69,11 @@ abstract static class Builder {
 
     abstract Builder setKeySet(KeySet keySet);
 
+    /**
+     * Note: {@link PartitionOptions} are currently ignored. See <a
+     * href="https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.v1#google.spanner.v1.PartitionOptions">
+     * PartitionOptions in RPC documents</a>
+     */
     abstract Builder setPartitionOptions(PartitionOptions partitionOptions);
 
     abstract ReadOperation build();
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java
index bdfc933b5c74..bb73a1c30872 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerAccessor.java
@@ -39,7 +39,6 @@
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.util.ReleaseInfo;
 import org.joda.time.Duration;
@@ -63,16 +62,12 @@ public class SpannerAccessor implements AutoCloseable {
   private static final ConcurrentHashMap<SpannerConfig, SpannerAccessor> spannerAccessors =
       new ConcurrentHashMap<>();
 
-  // Keep reference counts of each SpannerAccessor's usage so that we can close
-  // it when it is no longer in use.
-  private static final ConcurrentHashMap<SpannerConfig, AtomicInteger> refcounts =
-      new ConcurrentHashMap<>();
-
   private final Spanner spanner;
   private final DatabaseClient databaseClient;
   private final BatchClient batchClient;
   private final DatabaseAdminClient databaseAdminClient;
   private final SpannerConfig spannerConfig;
+  private int refcount = 0;
 
   private SpannerAccessor(
       Spanner spanner,
@@ -89,24 +84,20 @@ private SpannerAccessor(
 
   public static SpannerAccessor getOrCreate(SpannerConfig spannerConfig) {
 
-    SpannerAccessor self = spannerAccessors.get(spannerConfig);
-    if (self == null) {
-      synchronized (spannerAccessors) {
-        // Re-check that it has not been created before we got the lock.
-        self = spannerAccessors.get(spannerConfig);
-        if (self == null) {
-          // Connect to spanner for this SpannerConfig.
-          LOG.info("Connecting to {}", spannerConfig);
-          self = SpannerAccessor.createAndConnect(spannerConfig);
-          spannerAccessors.put(spannerConfig, self);
-          refcounts.putIfAbsent(spannerConfig, new AtomicInteger(0));
-        }
+    synchronized (spannerAccessors) {
+      SpannerAccessor self = spannerAccessors.get(spannerConfig);
+      if (self == null) {
+        // Connect to spanner for this SpannerConfig.
+        LOG.info("Connecting to {}", spannerConfig);
+        self = SpannerAccessor.createAndConnect(spannerConfig);
+        LOG.info("Successfully connected to {}", spannerConfig);
+        spannerAccessors.put(spannerConfig, self);
       }
+      // Add refcount for this spannerConfig.
+      self.refcount++;
+      LOG.debug("getOrCreate(): refcount={} for {}", self.refcount, spannerConfig);
+      return self;
     }
-    // Add refcount for this spannerConfig.
-    int refcount = refcounts.get(spannerConfig).incrementAndGet();
-    LOG.debug("getOrCreate(): refcount={} for {}", refcount, spannerConfig);
-    return self;
   }
 
   private static SpannerAccessor createAndConnect(SpannerConfig spannerConfig) {
@@ -261,18 +252,13 @@ public DatabaseAdminClient getDatabaseAdminClient() {
   @Override
   public void close() {
     // Only close Spanner when present in map and refcount == 0
-    int refcount = refcounts.getOrDefault(spannerConfig, new AtomicInteger(0)).decrementAndGet();
-    LOG.debug("close(): refcount={} for {}", refcount, spannerConfig);
-
-    if (refcount == 0) {
-      synchronized (spannerAccessors) {
-        // Re-check refcount in case it has increased outside the lock.
-        if (refcounts.get(spannerConfig).get() <= 0) {
-          spannerAccessors.remove(spannerConfig);
-          refcounts.remove(spannerConfig);
-          LOG.info("Closing {} ", spannerConfig);
-          spanner.close();
-        }
+    synchronized (spannerAccessors) {
+      refcount--;
+      LOG.debug("close(): refcount={} for {}", refcount, spannerConfig);
+      if (refcount <= 0) {
+        spannerAccessors.remove(spannerConfig);
+        LOG.info("Closing {} ", spannerConfig);
+        spanner.close();
       }
     }
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
index 786fa91f5582..3128de45fde3 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerIO.java
@@ -56,6 +56,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -102,9 +103,11 @@
 import org.apache.beam.sdk.transforms.Wait;
 import org.apache.beam.sdk.transforms.WithTimestamps;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.DefaultTrigger;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.transforms.windowing.Window;
 import org.apache.beam.sdk.util.BackOff;
 import org.apache.beam.sdk.util.FluentBackoff;
@@ -142,12 +145,15 @@
  *
  * <h3>Reading from Cloud Spanner</h3>
  *
- * <p>To read from Cloud Spanner, apply {@link Read} transformation. It will return a {@link
- * PCollection} of {@link Struct Structs}, where each element represents an individual row returned
- * from the read operation. Both Query and Read APIs are supported. See more information about <a
+ * <h4>Bulk reading of a single query or table</h4>
+ *
+ * <p>To perform a single read from Cloud Spanner, construct a {@link Read} transform using {@link
+ * SpannerIO#read() SpannerIO.read()}. It will return a {@link PCollection} of {@link Struct
+ * Structs}, where each element represents an individual row returned from the read operation. Both
+ * Query and Read APIs are supported. See more information about <a
  * href="https://cloud.google.com/spanner/docs/reads">reading from Cloud Spanner</a>
  *
- * <p>To execute a <strong>query</strong>, specify a {@link Read#withQuery(Statement)} or {@link
+ * <p>To execute a <strong>Query</strong>, specify a {@link Read#withQuery(Statement)} or {@link
  * Read#withQuery(String)} during the construction of the transform.
  *
  * <pre>{@code
@@ -158,8 +164,29 @@
  *         .withQuery("SELECT id, name, email FROM users"));
  * }</pre>
  *
- * <p>To use the Read API, specify a {@link Read#withTable(String) table name} and a {@link
- * Read#withColumns(List) list of columns}.
+ * <p>Reads by default use the <a
+ * href="https://cloud.google.com/spanner/docs/reads#read_data_in_parallel">PartitionQuery API</a>
+ * which enforces some limitations on the type of queries that can be used so that the data can be
+ * read in parallel. If the query is not supported by the PartitionQuery API, then you can specify a
+ * non-partitioned read by setting {@link Read#withBatching(boolean) withBatching(false)}. If the
+ * amount of data being read by a non-partitioned read is very large, it may be useful to add a
+ * {@link Reshuffle#viaRandomKey()} transform on the output so that the downstream transforms can
+ * run in parallel.
+ *
+ * <p>To read an entire <strong>Table</strong>, use {@link Read#withTable(String)} and optionally
+ * specify a {@link Read#withColumns(List) list of columns}.
+ *
+ * <pre>{@code
+ * PCollection<Struct> rows = p.apply(
+ *    SpannerIO.read()
+ *        .withInstanceId(instanceId)
+ *        .withDatabaseId(dbId)
+ *        .withTable("users")
+ *        .withColumns("id", "name", "email"));
+ * }</pre>
+ *
+ * <p>To read using an <strong>Index</strong>, specify the index name using {@link
+ * Read#withIndex(String)}.
  *
  * <pre>{@code
  * PCollection<Struct> rows = p.apply(
@@ -167,16 +194,17 @@
  *        .withInstanceId(instanceId)
  *        .withDatabaseId(dbId)
  *        .withTable("users")
+ *        .withIndex("users_by_name")
  *        .withColumns("id", "name", "email"));
  * }</pre>
  *
- * <p>To optimally read using index, specify the index name using {@link Read#withIndex}.
+ * <h4>Read consistency</h4>
  *
  * <p>The transform is guaranteed to be executed on a consistent snapshot of data, utilizing the
  * power of read only transactions. Staleness of data can be controlled using {@link
  * Read#withTimestampBound} or {@link Read#withTimestamp(Timestamp)} methods. <a
- * href="https://cloud.google.com/spanner/docs/transactions">Read more</a> about transactions in
- * Cloud Spanner.
+ * href="https://cloud.google.com/spanner/docs/transactions#read-only_transactions">Read more</a>
+ * about transactions in Cloud Spanner.
  *
  * <p>It is possible to read several {@link PCollection PCollections} within a single transaction.
  * Apply {@link SpannerIO#createTransaction()} transform, that lazily creates a transaction. The
@@ -204,6 +232,29 @@
  *        .withTransaction(tx));
  * }</pre>
  *
+ * <h4>Bulk reading of multiple queries or tables</h4>
+ *
+ * You can perform multiple consistent reads on a set of tables or using a set of queries by
+ * constructing a {@link ReadAll} transform using {@link SpannerIO#readAll() SpannerIO.readAll()}.
+ * This transform takes a {@link PCollection} of {@link ReadOperation} elements, and performs the
+ * partitioned read on each of them using the same Read Only Transaction for consistent results.
+ *
+ * <p>Note that this transform should <strong>not</strong> be used in Streaming pipelines. This is
+ * because the same Read Only Transaction, which is created once when the pipeline is first
+ * executed, will be used for all reads. The data being read will therefore become stale, and if no
+ * reads are made for more than 1 hour, the transaction will automatically timeout and be closed by
+ * the Spanner server, meaning that any subsequent reads will fail.
+ *
+ * <pre>{@code
+ * // Build a collection of ReadOperations.
+ * PCollection<ReadOperation> reads = ...
+ *
+ * PCollection<Struct> rows = reads.apply(
+ *     SpannerIO.readAll()
+ *         .withInstanceId(instanceId)
+ *         .withDatabaseId(dbId)
+ * }</pre>
+ *
  * <h3>Writing to Cloud Spanner</h3>
  *
  * <p>The Cloud Spanner {@link Write} transform writes to Cloud Spanner by executing a collection of
@@ -362,6 +413,12 @@
  * <p>{@link Write} can be used as a streaming sink, however as with batch mode note that the write
  * order of individual {@link Mutation}/{@link MutationGroup} objects is not guaranteed.
  *
+ * <p>{@link Read} and {@link ReadAll} can be used in Streaming pipelines to read a set of Facts on
+ * pipeline startup.
+ *
+ * <p>{@link ReadAll} should not be used on an unbounded {@code PCollection<ReadOperation>}, for the
+ * reasons stated above.
+ *
  * <h3>Updates to the I/O connector code</h3>
  *
  * For any significant significant updates to this I/O connector, please consider involving
@@ -564,8 +621,10 @@ public ReadAll withTimestampBound(TimestampBound timestampBound) {
     }
 
     /**
-     * By default Batch API is used to read data from Cloud Spanner. It is useful to disable
-     * batching when the underlying query is not root-partitionable.
+     * By default the <a
+     * href="https://cloud.google.com/spanner/docs/reads#read_data_in_parallel">PartitionQuery
+     * API</a> is used to read data from Cloud Spanner. It is useful to disable batching when the
+     * underlying query is not root-partitionable.
      */
     public ReadAll withBatching(boolean batching) {
       return toBuilder().setBatching(batching).build();
@@ -585,6 +644,15 @@ public ReadAll withHighPriority() {
 
     @Override
     public PCollection<Struct> expand(PCollection<ReadOperation> input) {
+
+      if (PCollection.IsBounded.UNBOUNDED == input.isBounded()) {
+        // Warn that SpannerIO.ReadAll should not be used on unbounded inputs.
+        LOG.warn(
+            "SpannerIO.ReadAll({}) is being applied to an unbounded input. "
+                + "This is not supported and can lead to runtime failures.",
+            this.getName());
+      }
+
       PTransform<PCollection<ReadOperation>, PCollection<Struct>> readTransform;
       if (getBatching()) {
         readTransform =
@@ -787,6 +855,11 @@ public Read withIndex(String index) {
       return withReadOperation(getReadOperation().withIndex(index));
     }
 
+    /**
+     * Note that {@link PartitionOptions} are currently ignored. See <a
+     * href="https://cloud.google.com/spanner/docs/reference/rpc/google.spanner.v1#google.spanner.v1.PartitionOptions">
+     * PartitionOptions in RPC documents</a>
+     */
     public Read withPartitionOptions(PartitionOptions partitionOptions) {
       return withReadOperation(getReadOperation().withPartitionOptions(partitionOptions));
     }
@@ -1932,6 +2005,15 @@ public void output(Iterable<MutationGroup> output) {
       public void outputWithTimestamp(Iterable<MutationGroup> output, Instant timestamp) {
         c.output(output, timestamp, GlobalWindow.INSTANCE);
       }
+
+      @Override
+      public void outputWindowedValue(
+          Iterable<MutationGroup> output,
+          Instant timestamp,
+          Collection<? extends BoundedWindow> windows,
+          PaneInfo paneInfo) {
+        throw new UnsupportedOperationException("outputWindowedValue not supported");
+      }
     }
   }
 
@@ -2100,6 +2182,7 @@ public void processElement(ProcessContext c) throws Exception {
           // fall through and retry individual mutationGroups.
         } else if (failureMode == FailureMode.FAIL_FAST) {
           mutationGroupsWriteFail.inc(mutations.size());
+          LOG.error("Failed to write a batch of mutation groups", e);
           throw e;
         } else {
           throw new IllegalArgumentException("Unknown failure mode " + failureMode);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java
index 38cd97da860a..809d7a275512 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/spanner/SpannerTransformRegistrar.java
@@ -288,6 +288,7 @@ public static class Configuration extends CrossLanguageConfiguration {
       private @Nullable Integer groupingFactor;
       private @Nullable Duration commitDeadline;
       private @Nullable Duration maxCumulativeBackoff;
+      private @Nullable String failureMode;
 
       public void setTable(String table) {
         this.table = table;
@@ -322,6 +323,10 @@ public void setMaxCumulativeBackoff(@Nullable Long maxCumulativeBackoff) {
           this.maxCumulativeBackoff = Duration.standardSeconds(maxCumulativeBackoff);
         }
       }
+
+      public void setFailureMode(@Nullable String failureMode) {
+        this.failureMode = failureMode;
+      }
     }
 
     @Override
@@ -361,6 +366,11 @@ public PTransform<PCollection<Row>, PDone> buildExternal(
         writeTransform =
             writeTransform.withMaxCumulativeBackoff(configuration.maxCumulativeBackoff);
       }
+      if (configuration.failureMode != null) {
+        writeTransform =
+            writeTransform.withFailureMode(
+                SpannerIO.FailureMode.valueOf(configuration.failureMode));
+      }
       return SpannerIO.WriteRows.of(writeTransform, operation, configuration.table);
     }
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java
index 5e6e3ac7ed07..c258ce4ab7fb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeBigQueryServices.java
@@ -43,7 +43,7 @@
 })
 public class FakeBigQueryServices implements BigQueryServices {
   private JobService jobService;
-  private DatasetService datasetService;
+  private FakeDatasetService datasetService;
   private StorageClient storageClient;
 
   public FakeBigQueryServices withJobService(JobService jobService) {
@@ -71,6 +71,11 @@ public DatasetService getDatasetService(BigQueryOptions bqOptions) {
     return datasetService;
   }
 
+  @Override
+  public WriteStreamService getWriteStreamService(BigQueryOptions bqOptions) {
+    return datasetService;
+  }
+
   @Override
   public StorageClient getStorageClient(BigQueryOptions bqOptions) {
     return storageClient;
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java
index f26c38d1e3c8..1e746d7f96be 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/FakeDatasetService.java
@@ -65,6 +65,7 @@
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.DatasetService;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.StreamAppendClient;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices.WriteStreamService;
 import org.apache.beam.sdk.io.gcp.bigquery.ErrorContainer;
 import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy;
 import org.apache.beam.sdk.io.gcp.bigquery.InsertRetryPolicy.Context;
@@ -84,7 +85,7 @@
 @SuppressWarnings({
   "nullness" // TODO(https://github.com/apache/beam/issues/20497)
 })
-public class FakeDatasetService implements DatasetService, Serializable {
+public class FakeDatasetService implements DatasetService, WriteStreamService, Serializable {
   // Table information must be static, as each ParDo will get a separate instance of
   // FakeDatasetServices, and they must all modify the same storage.
   static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Table<
@@ -679,7 +680,7 @@ public ApiFuture<AppendRowsResponse> appendRows(long offset, ProtoRows rows)
           }
           if (!rowIndexToErrorMessage.isEmpty()) {
             return ApiFutures.immediateFailedFuture(
-                new Exceptions.AppendSerializtionError(
+                new Exceptions.AppendSerializationError(
                     Code.INVALID_ARGUMENT.getNumber(),
                     "Append serialization failed for writer: " + streamName,
                     stream.streamName,
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java
index 1b3c844e2a9f..3ee6931dd98d 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryClusteringIT.java
@@ -27,6 +27,7 @@
 import java.util.Arrays;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.testing.BigqueryClient;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -68,7 +69,8 @@ public class BigQueryClusteringIT {
   public void setUp() {
     PipelineOptionsFactory.register(BigQueryClusteringITOptions.class);
     options = TestPipeline.testingPipelineOptions().as(BigQueryClusteringITOptions.class);
-    options.setTempLocation(options.getTempRoot() + "/temp-it/");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "temp-it").toString());
     bqClient = BigqueryClient.getNewBigqueryClient(options.getAppName());
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java
index 4b7e02e4bf47..9da050a059cc 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOJsonIT.java
@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.stream.Collectors;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
@@ -70,7 +71,8 @@ public class BigQueryIOJsonIT {
 
   static {
     TestPipelineOptions opt = TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class);
-    testOptions.setTempLocation(opt.getTempRoot() + "/java-tmp");
+    testOptions.setTempLocation(
+        FileSystems.matchNewDirectory(opt.getTempRoot(), "java-tmp").toString());
   }
 
   @Rule public final transient TestPipeline p = TestPipeline.fromOptions(testOptions);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java
index 8a3bc0fac2b7..6291eac62b33 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOReadIT.java
@@ -20,6 +20,7 @@
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.ExperimentalOptions;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
@@ -71,7 +72,8 @@ private void setupTestEnvironment(String recordSize) {
     PipelineOptionsFactory.register(BigQueryIOReadOptions.class);
     options = TestPipeline.testingPipelineOptions().as(BigQueryIOReadOptions.class);
     options.setNumRecords(numOfRecords.get(recordSize));
-    options.setTempLocation(options.getTempRoot() + "/temp-it/");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "temp-it").toString());
     project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
     options.setInputTable(project + ":" + datasetId + "." + tablePrefix + recordSize);
   }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java
index 35e2676c70ef..7998bac65055 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOStorageReadTableRowIT.java
@@ -24,6 +24,7 @@
 import java.util.Set;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TableRowParser;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method;
 import org.apache.beam.sdk.options.Description;
@@ -88,7 +89,8 @@ private void setUpTestEnvironment(String tableName) {
     options = TestPipeline.testingPipelineOptions().as(BigQueryIOStorageReadTableRowOptions.class);
     String project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
     options.setInputTable(project + ":" + DATASET_ID + "." + TABLE_PREFIX + tableName);
-    options.setTempLocation(options.getTempRoot() + "/temp-it/");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "temp-it").toString());
   }
 
   private static void runPipeline(BigQueryIOStorageReadTableRowOptions pipelineOptions) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTranslationTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTranslationTest.java
new file mode 100644
index 000000000000..c46d382bb298
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOTranslationTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigquery;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.google.api.services.bigquery.model.TableRow;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
+import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
+import org.junit.Test;
+
+public class BigQueryIOTranslationTest {
+
+  // A mapping from Read transform builder methods to the corresponding schema fields in
+  // KafkaIOTranslation.
+  static final Map<String, String> READ_TRANSFORM_SCHEMA_MAPPING = new HashMap<>();
+
+  static {
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getJsonTableRef", "json_table_ref");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getQuery", "query");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getValidate", "validate");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getFlattenResults", "flatten_results");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getUseLegacySql", "use_legacy_sql");
+    READ_TRANSFORM_SCHEMA_MAPPING.put(
+        "getWithTemplateCompatibility", "with_template_compatibility");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getBigQueryServices", "bigquery_services");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getParseFn", "parse_fn");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getDatumReaderFactory", "datum_reader_factory");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getQueryPriority", "query_priority");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getQueryLocation", "query_location");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getQueryTempDataset", "query_temp_dataset");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getMethod", "method");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getFormat", "format");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getSelectedFields", "selected_fields");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getRowRestriction", "row_restriction");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getCoder", "coder");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getKmsKey", "kms_key");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getTypeDescriptor", "type_descriptor");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getToBeamRowFn", "to_beam_row_fn");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getFromBeamRowFn", "from_beam_row_fn");
+    READ_TRANSFORM_SCHEMA_MAPPING.put("getUseAvroLogicalTypes", "use_avro_logical_types");
+    READ_TRANSFORM_SCHEMA_MAPPING.put(
+        "getProjectionPushdownApplied", "projection_pushdown_applied");
+  }
+
+  static final Map<String, String> WRITE_TRANSFORM_SCHEMA_MAPPING = new HashMap<>();
+
+  static {
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getJsonTableRef", "json_table_ref");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getTableFunction", "table_function");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getFormatFunction", "format_function");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put(
+        "getFormatRecordOnFailureFunction", "format_record_on_failure_function");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getAvroRowWriterFactory", "avro_row_writer_factory");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getAvroSchemaFactory", "avro_schema_factory");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getUseAvroLogicalTypes", "use_avro_logical_types");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getDynamicDestinations", "dynamic_destinations");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getJsonSchema", "json_schema");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getJsonTimePartitioning", "json_time_partitioning");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getClustering", "clustering");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getCreateDisposition", "create_disposition");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getWriteDisposition", "write_disposition");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getSchemaUpdateOptions", "schema_update_options");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getTableDescription", "table_description");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getValidate", "validate");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getBigQueryServices", "bigquery_services");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getMaxFilesPerBundle", "max_files_per_bundle");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getMaxFileSize", "max_file_size");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getNumFileShards", "num_file_shards");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put(
+        "getNumStorageWriteApiStreams", "num_storage_write_api_streams");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put(
+        "getPropagateSuccessfulStorageApiWrites", "propagate_successful_storage_api_writes");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getMaxFilesPerPartition", "max_files_per_partition");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getMaxBytesPerPartition", "max_bytes_per_partition");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getTriggeringFrequency", "triggering_frequency");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getMethod", "method");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getLoadJobProjectId", "load_job_project_id");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getFailedInsertRetryPolicy", "failed_insert_retry_policy");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getCustomGcsTempLocation", "custom_gcs_temp_location");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getExtendedErrorInfo", "extended_error_info");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getSkipInvalidRows", "skip_invalid_rows");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getIgnoreUnknownValues", "ignore_unknown_values");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getIgnoreInsertIds", "ignore_insert_ids");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getMaxRetryJobs", "max_retry_jobs");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getKmsKey", "kms_key");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getPrimaryKey", "primary_key");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put(
+        "getDefaultMissingValueInterpretation", "default_missing_value_interpretation");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getOptimizeWrites", "optimize_writes");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getUseBeamSchema", "use_beam_schema");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getAutoSharding", "auto_sharding");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getPropagateSuccessful", "propagate_successful");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getAutoSchemaUpdate", "auto_schema_update");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getWriteProtosClass", "write_protos_class");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getDirectWriteProtos", "direct_write_protos");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getDeterministicRecordIdFn", "deterministic_record_id_fn");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put("getWriteTempDataset", "write_temp_dataset");
+    WRITE_TRANSFORM_SCHEMA_MAPPING.put(
+        "getRowMutationInformationFn", "row_mutation_information_fn");
+  }
+
+  @Test
+  public void testReCreateReadTransformFromRowTable() {
+    // setting a subset of fields here.
+    BigQueryIO.TypedRead<TableRow> readTransform =
+        BigQueryIO.readTableRows()
+            .from("dummyproject:dummydataset.dummytable")
+            .withMethod(TypedRead.Method.DIRECT_READ)
+            .withKmsKey("dummykmskey")
+            .withTemplateCompatibility();
+
+    BigQueryIOTranslation.BigQueryIOReadTranslator translator =
+        new BigQueryIOTranslation.BigQueryIOReadTranslator();
+    Row row = translator.toConfigRow(readTransform);
+
+    BigQueryIO.TypedRead<TableRow> readTransformFromRow =
+        (BigQueryIO.TypedRead<TableRow>) translator.fromConfigRow(row);
+    assertNotNull(readTransformFromRow.getTable());
+    assertEquals("dummyproject", readTransformFromRow.getTable().getProjectId());
+    assertEquals("dummydataset", readTransformFromRow.getTable().getDatasetId());
+    assertEquals("dummytable", readTransformFromRow.getTable().getTableId());
+    assertEquals(TypedRead.Method.DIRECT_READ, readTransformFromRow.getMethod());
+    assertEquals("dummykmskey", readTransformFromRow.getKmsKey());
+    assertTrue(readTransformFromRow.getWithTemplateCompatibility());
+  }
+
+  static class DummyParseFn implements SerializableFunction<SchemaAndRecord, Object> {
+    @Override
+    public Object apply(SchemaAndRecord input) {
+      return null;
+    }
+  }
+
+  @Test
+  public void testReCreateReadTransformFromRowQuery() {
+    // setting a subset of fields here.
+    BigQueryIO.TypedRead<?> readTransform =
+        BigQueryIO.read(new DummyParseFn())
+            .fromQuery("dummyquery")
+            .useAvroLogicalTypes()
+            .usingStandardSql();
+
+    BigQueryIOTranslation.BigQueryIOReadTranslator translator =
+        new BigQueryIOTranslation.BigQueryIOReadTranslator();
+    Row row = translator.toConfigRow(readTransform);
+
+    BigQueryIO.TypedRead<?> readTransformFromRow = translator.fromConfigRow(row);
+    assertEquals("dummyquery", readTransformFromRow.getQuery().get());
+    assertNotNull(readTransformFromRow.getParseFn());
+    assertTrue(readTransformFromRow.getParseFn() instanceof DummyParseFn);
+    assertTrue(readTransformFromRow.getUseAvroLogicalTypes());
+    assertFalse(readTransformFromRow.getUseLegacySql());
+  }
+
+  @Test
+  public void testReadTransformRowIncludesAllFields() {
+    // These fields do not represent properties of the transform.
+    List<String> fieldsToIgnore =
+        ImmutableList.of("getFinalSchema", "getTableProvider", "getTable");
+
+    List<String> getMethodNames =
+        Arrays.stream(BigQueryIO.TypedRead.class.getDeclaredMethods())
+            .map(
+                method -> {
+                  return method.getName();
+                })
+            .filter(methodName -> methodName.startsWith("get"))
+            .filter(methodName -> !fieldsToIgnore.contains(methodName))
+            .collect(Collectors.toList());
+
+    // Just to make sure that this does not pass trivially.
+    assertTrue(getMethodNames.size() > 0);
+
+    for (String getMethodName : getMethodNames) {
+      assertTrue(
+          "Method "
+              + getMethodName
+              + " will not be tracked when upgrading the 'BigQueryIO.TypedRead' transform. Please update"
+              + "'BigQueryIOTranslation.BigQueryIOReadTranslator' to track the new method "
+              + "and update this test.",
+          READ_TRANSFORM_SCHEMA_MAPPING.keySet().contains(getMethodName));
+    }
+
+    // Confirming that all fields mentioned in `READ_TRANSFORM_SCHEMA_MAPPING` are
+    // actually available in the schema.
+    READ_TRANSFORM_SCHEMA_MAPPING.values().stream()
+        .forEach(
+            fieldName -> {
+              assertTrue(
+                  "Field name "
+                      + fieldName
+                      + " was not found in the transform schema defined in "
+                      + "BigQueryIOTranslation.BigQueryIOReadTranslator.",
+                  BigQueryIOTranslation.BigQueryIOReadTranslator.schema
+                      .getFieldNames()
+                      .contains(fieldName));
+            });
+  }
+
+  @Test
+  public void testReCreateWriteTransformFromRowTable() {
+    // setting a subset of fields here.
+    BigQueryIO.Write<?> writeTransform =
+        BigQueryIO.write()
+            .to("dummyproject:dummydataset.dummytable")
+            .withAutoSharding()
+            .withTriggeringFrequency(org.joda.time.Duration.millis(10000))
+            .withWriteDisposition(WriteDisposition.WRITE_TRUNCATE)
+            .withCreateDisposition(CreateDisposition.CREATE_NEVER)
+            .withKmsKey("dummykmskey");
+
+    BigQueryIOTranslation.BigQueryIOWriteTranslator translator =
+        new BigQueryIOTranslation.BigQueryIOWriteTranslator();
+    Row row = translator.toConfigRow(writeTransform);
+
+    BigQueryIO.Write<?> writeTransformFromRow = (BigQueryIO.Write<?>) translator.fromConfigRow(row);
+    assertNotNull(writeTransformFromRow.getTable());
+    assertEquals("dummyproject", writeTransformFromRow.getTable().get().getProjectId());
+    assertEquals("dummydataset", writeTransformFromRow.getTable().get().getDatasetId());
+    assertEquals("dummytable", writeTransformFromRow.getTable().get().getTableId());
+    assertEquals(WriteDisposition.WRITE_TRUNCATE, writeTransformFromRow.getWriteDisposition());
+    assertEquals(CreateDisposition.CREATE_NEVER, writeTransformFromRow.getCreateDisposition());
+    assertEquals("dummykmskey", writeTransformFromRow.getKmsKey());
+  }
+
+  @Test
+  public void testWriteTransformRowIncludesAllFields() {
+    // These fields do not represent properties of the transform.
+    List<String> fieldsToIgnore =
+        ImmutableList.of(
+            "getSchemaFromView",
+            "getStorageApiNumStreams",
+            "getStorageApiTriggeringFrequency",
+            "getTableWithDefaultProject",
+            "getTable");
+
+    List<String> getMethodNames =
+        Arrays.stream(BigQueryIO.Write.class.getDeclaredMethods())
+            .map(
+                method -> {
+                  return method.getName();
+                })
+            .filter(methodName -> methodName.startsWith("get"))
+            .filter(methodName -> !fieldsToIgnore.contains(methodName))
+            .collect(Collectors.toList());
+
+    // Just to make sure that this does not pass trivially.
+    assertTrue(getMethodNames.size() > 0);
+
+    for (String getMethodName : getMethodNames) {
+      assertTrue(
+          "Method "
+              + getMethodName
+              + " will not be tracked when upgrading the 'BigQueryIO.Write' transform. Please update"
+              + "'BigQueryIOTranslation.BigQueryIOWriteTranslator' to track the new method "
+              + "and update this test.",
+          WRITE_TRANSFORM_SCHEMA_MAPPING.keySet().contains(getMethodName));
+    }
+
+    // Confirming that all fields mentioned in `WRITE_TRANSFORM_SCHEMA_MAPPING` are
+    // actually available in the schema.
+    WRITE_TRANSFORM_SCHEMA_MAPPING.values().stream()
+        .forEach(
+            fieldName -> {
+              assertTrue(
+                  "Field name "
+                      + fieldName
+                      + " was not found in the transform schema defined in "
+                      + "BigQueryIOTranslation.BigQueryIOWriteTranslator.",
+                  BigQueryIOTranslation.BigQueryIOWriteTranslator.schema
+                      .getFieldNames()
+                      .contains(fieldName));
+            });
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
index 720419f2227a..55269342155f 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java
@@ -35,6 +35,8 @@
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 
+import com.google.api.core.ApiFuture;
+import com.google.api.core.SettableApiFuture;
 import com.google.api.services.bigquery.model.Clustering;
 import com.google.api.services.bigquery.model.ErrorProto;
 import com.google.api.services.bigquery.model.Job;
@@ -48,7 +50,12 @@
 import com.google.api.services.bigquery.model.TableSchema;
 import com.google.api.services.bigquery.model.TimePartitioning;
 import com.google.auto.value.AutoValue;
+import com.google.cloud.bigquery.storage.v1.AppendRowsRequest;
+import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
+import com.google.cloud.bigquery.storage.v1.Exceptions;
+import com.google.cloud.bigquery.storage.v1.ProtoRows;
 import com.google.protobuf.ByteString;
+import com.google.protobuf.DescriptorProtos;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
@@ -124,14 +131,18 @@
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.PeriodicImpulse;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.SerializableFunctions;
 import org.apache.beam.sdk.transforms.SimpleFunction;
+import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.View;
 import org.apache.beam.sdk.transforms.WithKeys;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.windowing.AfterWatermark;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
+import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.IncompatibleWindowException;
 import org.apache.beam.sdk.transforms.windowing.NonMergingWindowFn;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -3104,6 +3115,104 @@ public void testStorageWriteWithMultipleAppendsPerStream() throws Exception {
         containsInAnyOrder(Iterables.toArray(rows, TableRow.class)));
   }
 
+  public static class ThrowingFakeDatasetServices extends FakeDatasetService {
+    @Override
+    public BigQueryServices.StreamAppendClient getStreamAppendClient(
+        String streamName,
+        DescriptorProtos.DescriptorProto descriptor,
+        boolean useConnectionPool,
+        AppendRowsRequest.MissingValueInterpretation missingValueInterpretation) {
+      return new BigQueryServices.StreamAppendClient() {
+        @Override
+        public ApiFuture<AppendRowsResponse> appendRows(long offset, ProtoRows rows) {
+          Map<Integer, String> errorMap = new HashMap<>();
+          for (int i = 0; i < rows.getSerializedRowsCount(); i++) {
+            errorMap.put(i, "some serialization error");
+          }
+          SettableApiFuture<AppendRowsResponse> appendResult = SettableApiFuture.create();
+          appendResult.setException(
+              new Exceptions.AppendSerializationError(
+                  404, "some description", "some stream", errorMap));
+          return appendResult;
+        }
+
+        @Override
+        public com.google.cloud.bigquery.storage.v1.@Nullable TableSchema getUpdatedSchema() {
+          return null;
+        }
+
+        @Override
+        public void pin() {}
+
+        @Override
+        public void unpin() {}
+
+        @Override
+        public void close() {}
+      };
+    }
+  }
+
+  @Test
+  public void testStorageWriteReturnsAppendSerializationError() throws Exception {
+    assumeTrue(useStorageApi);
+    assumeTrue(useStreaming);
+    p.getOptions().as(BigQueryOptions.class).setStorageApiAppendThresholdRecordCount(5);
+
+    TableSchema schema =
+        new TableSchema()
+            .setFields(Arrays.asList(new TableFieldSchema().setType("INTEGER").setName("long")));
+    Table fakeTable = new Table();
+    TableReference ref =
+        new TableReference()
+            .setProjectId("project-id")
+            .setDatasetId("dataset-id")
+            .setTableId("table-id");
+    fakeTable.setSchema(schema);
+    fakeTable.setTableReference(ref);
+
+    ThrowingFakeDatasetServices throwingService = new ThrowingFakeDatasetServices();
+    throwingService.createTable(fakeTable);
+
+    int numRows = 100;
+
+    WriteResult res =
+        p.apply(
+                PeriodicImpulse.create()
+                    .startAt(Instant.ofEpochMilli(0))
+                    .stopAfter(Duration.millis(numRows - 1))
+                    .withInterval(Duration.millis(1)))
+            .apply(
+                "Convert to longs",
+                MapElements.into(TypeDescriptor.of(TableRow.class))
+                    .via(instant -> new TableRow().set("long", instant.getMillis())))
+            .apply(
+                BigQueryIO.writeTableRows()
+                    .to(ref)
+                    .withSchema(schema)
+                    .withTestServices(
+                        new FakeBigQueryServices()
+                            .withDatasetService(throwingService)
+                            .withJobService(fakeJobService)));
+
+    PCollection<Integer> numErrors =
+        res.getFailedStorageApiInserts()
+            .apply(
+                "Count errors",
+                MapElements.into(TypeDescriptors.integers())
+                    .via(err -> err.getErrorMessage().equals("some serialization error") ? 1 : 0))
+            .apply(
+                Window.<Integer>into(new GlobalWindows())
+                    .triggering(AfterWatermark.pastEndOfWindow())
+                    .discardingFiredPanes()
+                    .withAllowedLateness(Duration.ZERO))
+            .apply(Sum.integersGlobally());
+
+    PAssert.that(numErrors).containsInAnyOrder(numRows);
+
+    p.run().waitUntilFinish();
+  }
+
   @Test
   public void testWriteProtos() throws Exception {
     BigQueryIO.Write.Method method =
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java
index 85a180932f65..edf93dba3ae4 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java
@@ -26,6 +26,7 @@
 import java.security.SecureRandom;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.Method;
 import org.apache.beam.sdk.io.gcp.testing.BigqueryClient;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -87,7 +88,8 @@ private void testQueryAndWrite(Method method) throws Exception {
     String outputTableId = "testQueryAndWrite_" + method.name();
     String outputTableSpec = project + ":" + BIG_QUERY_DATASET_ID + "." + outputTableId;
 
-    options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "bq_it_temp").toString());
     Pipeline p = Pipeline.create(options);
     // Reading triggers BQ query and extract jobs. Writing triggers either a load job or performs a
     // streaming insert (depending on method).
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryNestedRecordsIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryNestedRecordsIT.java
index b85dc62c5fe9..e8d3a0ea22c4 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryNestedRecordsIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryNestedRecordsIT.java
@@ -19,6 +19,7 @@
 
 import com.google.api.services.bigquery.model.TableRow;
 import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.testing.BigqueryClient;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -63,7 +64,8 @@ public void testNestedRecords() throws Exception {
     TestPipelineOptions testOptions =
         TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class);
     Options options = testOptions.as(Options.class);
-    options.setTempLocation(testOptions.getTempRoot() + "/temp-it/");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(testOptions.getTempRoot(), "temp-it").toString());
     runPipeline(options);
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java
index 833a0a0829c7..c5e954a46380 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySchemaUpdateOptionsIT.java
@@ -38,6 +38,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.SchemaUpdateOption;
@@ -150,7 +151,8 @@ private void runWriteTest(
       List<List<String>> expectedResult)
       throws Exception {
     Options options = TestPipeline.testingPipelineOptions().as(Options.class);
-    options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "bq_it_temp").toString());
 
     Pipeline p = Pipeline.create(options);
     Create.Values<TableRow> input = Create.<TableRow>of(rowToInsert);
@@ -264,7 +266,8 @@ public void runWriteTestTempTableAndDynamicDestination() throws Exception {
     }
 
     Options options = TestPipeline.testingPipelineOptions().as(Options.class);
-    options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "bq_it_temp").toString());
 
     Pipeline p = Pipeline.create(options);
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
index 474aea020f69..a8e1ad52237c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImplTest.java
@@ -473,8 +473,7 @@ public void testGetTableSucceeds() throws Exception {
         });
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     Table table =
         datasetService.getTable(
@@ -509,7 +508,7 @@ public void testGetTableNullProjectSucceeds() throws Exception {
     options.setBigQueryProject("projectId");
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, null, options);
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, options);
 
     Table table =
         datasetService.getTable(
@@ -528,8 +527,7 @@ public void testGetTableNotFound() throws IOException, InterruptedException {
         });
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     TableReference tableRef =
         new TableReference()
@@ -562,8 +560,7 @@ public void testGetTableThrows() throws Exception {
     thrown.expectMessage(String.format("Unable to get table: %s", tableRef.getTableId()));
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     datasetService.getTable(
         tableRef, Collections.emptyList(), null, BackOff.STOP_BACKOFF, Sleeper.DEFAULT);
   }
@@ -593,8 +590,7 @@ public void testIsTableEmptySucceeds() throws Exception {
         });
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     assertFalse(datasetService.isTableEmpty(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT));
 
@@ -610,8 +606,7 @@ public void testIsTableEmptyNoRetryForNotFound() throws IOException, Interrupted
         });
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     TableReference tableRef =
         new TableReference()
@@ -644,8 +639,7 @@ public void testIsTableEmptyThrows() throws Exception {
             .setTableId("tableId");
 
     BigQueryServicesImpl.DatasetServiceImpl datasetService =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     thrown.expect(IOException.class);
     thrown.expectMessage(String.format("Unable to list table data: %s", tableRef.getTableId()));
@@ -715,7 +709,7 @@ public void testInsertRateLimitRetry() throws Exception {
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     dataService.insertAll(
         ref,
         rows,
@@ -760,7 +754,7 @@ public void testInsertQuotaExceededRetry() throws Exception {
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     dataService.insertAll(
         ref,
         rows,
@@ -817,7 +811,7 @@ public void testInsertStoppedRetry() throws Exception {
     thrown.expectMessage("quotaExceeded");
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     dataService.insertAll(
         ref,
         rows,
@@ -876,7 +870,7 @@ public void testInsertRetrySelectRows() throws Exception {
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     dataService.insertAll(
         ref,
         rows,
@@ -933,7 +927,7 @@ public void testInsertWithinRowCountLimits() throws Exception {
         PipelineOptionsFactory.fromArgs("--maxStreamingRowsToBatch=1").create();
     options.as(GcsOptions.class).setExecutorService(Executors.newSingleThreadExecutor());
 
-    DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, options);
+    DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, options);
     dataService.insertAll(
         ref,
         rows,
@@ -983,7 +977,7 @@ public void testInsertWithinRequestByteSizeLimitsErrorsOut() throws Exception {
 
     DatasetServiceImpl dataService =
         new DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create());
+            bigquery, PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create());
     List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
     List<ValueInSingleWindow<TableRow>> successfulRows = Lists.newArrayList();
     RuntimeException e =
@@ -1031,7 +1025,7 @@ public void testInsertRetryTransientsAboveRequestByteSizeLimits() throws Excepti
 
     DatasetServiceImpl dataService =
         new DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create());
+            bigquery, PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create());
     List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
     List<ValueInSingleWindow<TableRow>> successfulRows = Lists.newArrayList();
     dataService.<TableRow>insertAll(
@@ -1086,7 +1080,7 @@ public void testInsertWithinRequestByteSizeLimits() throws Exception {
         PipelineOptionsFactory.fromArgs("--maxStreamingBatchSize=15").create();
     options.as(GcsOptions.class).setExecutorService(Executors.newSingleThreadExecutor());
 
-    DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, null, options);
+    DatasetServiceImpl dataService = new DatasetServiceImpl(bigquery, options);
     dataService.insertAll(
         ref,
         rows,
@@ -1150,7 +1144,7 @@ public void testInsertFailsGracefully() throws Exception {
         row0FailureResponseFunction);
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     // Expect it to fail.
     try {
@@ -1209,7 +1203,7 @@ public void testFailInsertOtherRetry() throws Exception {
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     thrown.expect(RuntimeException.class);
     thrown.expectMessage("actually forbidden");
     try {
@@ -1258,7 +1252,7 @@ public void testInsertTimeoutLog() throws Exception {
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     RuntimeException e =
         assertThrows(
             RuntimeException.class,
@@ -1340,7 +1334,7 @@ public void testInsertRetryPolicy() throws InterruptedException, IOException {
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
     dataService.insertAll(
@@ -1387,7 +1381,7 @@ public void testSkipInvalidRowsIgnoreUnknownIgnoreInsertIdsValuesStreaming()
     setupMockResponses(allRowsSucceededResponseFunction, allRowsSucceededResponseFunction);
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     // First, test with all flags disabled
     dataService.insertAll(
@@ -1501,8 +1495,7 @@ public void testCreateTableSucceeds() throws IOException {
         });
 
     BigQueryServicesImpl.DatasetServiceImpl services =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     Table ret =
         services.tryCreateTable(
             testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 0), Sleeper.DEFAULT);
@@ -1535,8 +1528,7 @@ public void testCreateTableDoesNotRetry() throws IOException {
     thrown.expectMessage("actually forbidden");
 
     BigQueryServicesImpl.DatasetServiceImpl services =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     try {
       services.tryCreateTable(
           testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 3), Sleeper.DEFAULT);
@@ -1572,8 +1564,7 @@ public void testCreateTableSucceedsAlreadyExists() throws IOException {
         });
 
     BigQueryServicesImpl.DatasetServiceImpl services =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     Table ret =
         services.tryCreateTable(
             testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 0), Sleeper.DEFAULT);
@@ -1604,8 +1595,7 @@ public void testCreateTableRetry() throws IOException {
         });
 
     BigQueryServicesImpl.DatasetServiceImpl services =
-        new BigQueryServicesImpl.DatasetServiceImpl(
-            bigquery, null, PipelineOptionsFactory.create());
+        new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
     Table ret =
         services.tryCreateTable(
             testTable, new RetryBoundedBackOff(BackOff.ZERO_BACKOFF, 3), Sleeper.DEFAULT);
@@ -1651,7 +1641,7 @@ public void testSimpleErrorRetrieval() throws InterruptedException, IOException
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
     dataService.insertAll(
@@ -1711,7 +1701,7 @@ public void testExtendedErrorRetrieval() throws InterruptedException, IOExceptio
         });
 
     DatasetServiceImpl dataService =
-        new DatasetServiceImpl(bigquery, null, PipelineOptionsFactory.create());
+        new DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
 
     List<ValueInSingleWindow<BigQueryInsertError>> failedInserts = Lists.newArrayList();
     dataService.insertAll(
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetricsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetricsTest.java
index c937c6b299b2..9c6fae164fca 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetricsTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQuerySinkMetricsTest.java
@@ -20,6 +20,8 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
 
 import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
 import com.google.cloud.bigquery.storage.v1.Exceptions;
@@ -36,6 +38,7 @@
 import org.apache.beam.sdk.metrics.MetricsEnvironment;
 import org.apache.beam.sdk.util.HistogramData;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Lists;
 import org.hamcrest.collection.IsMapContaining;
 import org.junit.Test;
@@ -141,7 +144,7 @@ public void testThrowableToGRPCCodeString() throws Exception {
 
     int notFoundVal = Status.Code.NOT_FOUND.value();
     Throwable grpcError =
-        new Exceptions.AppendSerializtionError(notFoundVal, "Test Error", "Stream name", null);
+        new Exceptions.AppendSerializationError(notFoundVal, "Test Error", "Stream name", null);
     assertThat(BigQuerySinkMetrics.throwableToGRPCCodeString(grpcError), equalTo("NOT_FOUND"));
   }
 
@@ -217,7 +220,7 @@ public void testReportFailedRPCMetrics_KnownGrpcError() throws Exception {
     c.setOperationEndTime(t1.plusMillis(5));
     int notFoundVal = Status.Code.NOT_FOUND.value();
     Throwable grpcError =
-        new Exceptions.AppendSerializtionError(notFoundVal, "Test Error", "Stream name", null);
+        new Exceptions.AppendSerializationError(notFoundVal, "Test Error", "Stream name", null);
     c.setError(grpcError);
 
     // Test disabled SupportMetricsDeletion
@@ -301,4 +304,45 @@ public void testReportFailedRPCMetrics_UnknownGrpcError() throws Exception {
         testContainer.perWorkerHistograms.get(KV.of(histogramName, bucketType)).values,
         containsInAnyOrder(Double.valueOf(15.0)));
   }
+
+  @Test
+  public void testParseMetricName_noLabels() {
+    String baseMetricName = "baseMetricName";
+    BigQuerySinkMetrics.ParsedMetricName metricName =
+        BigQuerySinkMetrics.ParsedMetricName.create(baseMetricName);
+
+    assertThat(BigQuerySinkMetrics.parseMetricName(baseMetricName), equalTo(metricName));
+  }
+
+  @Test
+  public void testParseMetricName_successfulLabels() {
+    String metricName = "baseLabel-key1:val1;key2:val2;key3:val3;";
+    ImmutableMap<String, String> metricLabels =
+        ImmutableMap.of("key1", "val1", "key2", "val2", "key3", "val3");
+    BigQuerySinkMetrics.ParsedMetricName expectedName =
+        BigQuerySinkMetrics.ParsedMetricName.create("baseLabel", metricLabels);
+
+    assertThat(BigQuerySinkMetrics.parseMetricName(metricName), equalTo(expectedName));
+  }
+
+  @Test
+  public void testParseMetricName_malformedMetricName() {
+    String malformedMetricName = "baseLabel-key1:val1-key2:val2";
+    assertThat(BigQuerySinkMetrics.parseMetricName(malformedMetricName), is(nullValue()));
+  }
+
+  @Test
+  public void testParseMetricName_malformedMetricLabels() {
+    String metricName = "baseLabel-key1:val1:malformedField;key2:val2;";
+    ImmutableMap<String, String> metricLabels = ImmutableMap.of("key2", "val2");
+    BigQuerySinkMetrics.ParsedMetricName expectedName =
+        BigQuerySinkMetrics.ParsedMetricName.create("baseLabel", metricLabels);
+
+    assertThat(BigQuerySinkMetrics.parseMetricName(metricName), equalTo(expectedName));
+  }
+
+  @Test
+  public void testParseMetricName_emptyString() {
+    assertThat(BigQuerySinkMetrics.parseMetricName(""), is(nullValue()));
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java
index da5f396e8d89..22e4feb3c050 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryTimePartitioningClusteringIT.java
@@ -29,6 +29,7 @@
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.coders.Coder;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.testing.BigqueryClient;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -90,7 +91,8 @@ public static void setupTestEnvironment() throws Exception {
   public void setUp() {
     PipelineOptionsFactory.register(BigQueryClusteringITOptions.class);
     options = TestPipeline.testingPipelineOptions().as(BigQueryClusteringITOptions.class);
-    options.setTempLocation(options.getTempRoot() + "/temp-it/");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "temp-it").toString());
     bqClient = BigqueryClient.getNewBigqueryClient(options.getAppName());
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java
index 1abe7752b2e0..31bb627e8451 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryToTableIT.java
@@ -39,6 +39,7 @@
 import org.apache.beam.sdk.coders.VoidCoder;
 import org.apache.beam.sdk.extensions.gcp.options.GcpOptions;
 import org.apache.beam.sdk.extensions.gcp.util.BackOffAdapter;
+import org.apache.beam.sdk.io.FileSystems;
 import org.apache.beam.sdk.io.gcp.testing.BigqueryClient;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
@@ -118,7 +119,8 @@ private void runBigQueryToTablePipeline(BigQueryToTableOptions options) {
   private BigQueryToTableOptions setupLegacyQueryTest(String outputTable) {
     BigQueryToTableOptions options =
         TestPipeline.testingPipelineOptions().as(BigQueryToTableOptions.class);
-    options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "bq_it_temp").toString());
     options.setQuery("SELECT * FROM (SELECT \"apple\" as fruit), (SELECT \"orange\" as fruit),");
     options.setOutput(outputTable);
     options.setOutputSchema(BigQueryToTableIT.LEGACY_QUERY_TABLE_SCHEMA);
@@ -128,7 +130,8 @@ private BigQueryToTableOptions setupLegacyQueryTest(String outputTable) {
   private BigQueryToTableOptions setupNewTypesQueryTest(String outputTable) {
     BigQueryToTableOptions options =
         TestPipeline.testingPipelineOptions().as(BigQueryToTableOptions.class);
-    options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "bq_it_temp").toString());
     options.setQuery(
         String.format(
             "SELECT bytes, date, time FROM [%s:%s.%s]",
@@ -140,7 +143,8 @@ private BigQueryToTableOptions setupNewTypesQueryTest(String outputTable) {
 
   private BigQueryToTableOptions setupStandardQueryTest(String outputTable) {
     BigQueryToTableOptions options = this.setupLegacyQueryTest(outputTable);
-    options.setTempLocation(options.getTempRoot() + "/bq_it_temp");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "bq_it_temp").toString());
     options.setQuery(
         "SELECT * FROM (SELECT \"apple\" as fruit) UNION ALL (SELECT \"orange\" as fruit)");
     options.setUsingStandardSql(true);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
index cef93913401e..c71946a54676 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryUtilTest.java
@@ -170,7 +170,7 @@ public void testTableGet() throws InterruptedException, IOException {
     onTableList(dataList);
 
     BigQueryServicesImpl.DatasetServiceImpl services =
-        new BigQueryServicesImpl.DatasetServiceImpl(mockClient, null, options);
+        new BigQueryServicesImpl.DatasetServiceImpl(mockClient, options);
 
     services.getTable(
         new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table"));
@@ -190,7 +190,7 @@ public void testInsertAll() throws Exception {
     onInsertAll(errorsIndices);
 
     TableReference ref = BigQueryHelpers.parseTableSpec("project:dataset.table");
-    DatasetServiceImpl datasetService = new DatasetServiceImpl(mockClient, null, options, 5);
+    DatasetServiceImpl datasetService = new DatasetServiceImpl(mockClient, options, 5);
 
     List<FailsafeValueInSingleWindow<TableRow, TableRow>> rows = new ArrayList<>();
     List<String> ids = new ArrayList<>();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
index 714dc9f8619d..6cda518eb124 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIOTest.java
@@ -1695,8 +1695,8 @@ public FakeBigtableReader createReader(BigtableSource source) {
     }
 
     @Override
-    public FakeBigtableWriter openForWriting(String tableId) {
-      return new FakeBigtableWriter(tableId);
+    public FakeBigtableWriter openForWriting(BigtableWriteOptions writeOptions) {
+      return new FakeBigtableWriter(writeOptions.getTableId().get());
     }
 
     @Override
@@ -1748,8 +1748,8 @@ public FakeBigtableReader createReader(BigtableSource source) {
     }
 
     @Override
-    public FailureBigtableWriter openForWriting(String tableId) {
-      return new FailureBigtableWriter(tableId, this, failureOptions);
+    public FailureBigtableWriter openForWriting(BigtableWriteOptions writeOptions) {
+      return new FailureBigtableWriter(writeOptions.getTableId().get(), this, failureOptions);
     }
 
     @Override
@@ -1929,9 +1929,6 @@ public CompletionStage<MutateRowResponse> writeRecord(KV<ByteString, Iterable<Mu
       return CompletableFuture.completedFuture(MutateRowResponse.getDefaultInstance());
     }
 
-    @Override
-    public void flush() {}
-
     @Override
     public void close() {}
   }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java
index f239a68462d8..9a07f625ca1c 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableServiceImplTest.java
@@ -594,6 +594,69 @@ public void testReadFillBuffer() throws IOException {
     Mockito.verify(mockCallMetric, Mockito.times(3)).call("ok");
   }
 
+  /** This test verifies that a refill request is sent before we read all the rows in the buffer. */
+  @Test
+  public void testRefillOnLowWatermark() throws IOException {
+    int segmentSize = 30;
+    RowSet.Builder ranges = RowSet.newBuilder();
+    // generate 3 pages of rows
+    ranges.addRowRanges(
+        generateRowRange(
+            generateByteString(DEFAULT_PREFIX, 0),
+            generateByteString(DEFAULT_PREFIX, segmentSize * 3)));
+
+    List<List<Row>> expectedResults =
+        ImmutableList.of(
+            generateSegmentResult(DEFAULT_PREFIX, 0, segmentSize),
+            generateSegmentResult(DEFAULT_PREFIX, segmentSize, segmentSize),
+            generateSegmentResult(DEFAULT_PREFIX, segmentSize * 2, segmentSize),
+            ImmutableList.of());
+
+    // Set up Callable to be returned by stub.createReadRowsCallable()
+    ServerStreamingCallable<Query, Row> mockCallable = Mockito.mock(ServerStreamingCallable.class);
+    // Return multiple answers when mockCallable is called
+    doAnswer(new MultipleAnswer<Row>(expectedResults))
+        .when(mockCallable)
+        .call(any(Query.class), any(ResponseObserver.class), any(ApiCallContext.class));
+    when(mockStub.createReadRowsCallable(any(RowAdapter.class))).thenReturn(mockCallable);
+    ServerStreamingCallable<Query, Row> callable =
+        mockStub.createReadRowsCallable(new BigtableServiceImpl.BigtableRowProtoAdapter());
+    // Set up client to return callable
+    when(mockBigtableDataClient.readRowsCallable(any(RowAdapter.class))).thenReturn(callable);
+
+    RetrySettings retrySettings =
+        bigtableDataSettings.getStubSettings().bulkReadRowsSettings().getRetrySettings();
+    BigtableService.Reader underTest =
+        new BigtableServiceImpl.BigtableSegmentReaderImpl(
+            mockBigtableDataClient,
+            bigtableDataSettings.getProjectId(),
+            bigtableDataSettings.getInstanceId(),
+            TABLE_ID,
+            ranges.build(),
+            RowFilter.getDefaultInstance(),
+            segmentSize,
+            DEFAULT_BYTE_SEGMENT_SIZE,
+            Duration.millis(retrySettings.getInitialRpcTimeout().toMillis()),
+            Duration.millis(retrySettings.getTotalTimeout().toMillis()),
+            mockCallMetric);
+
+    Assert.assertTrue(underTest.start());
+
+    int refillWatermark = Math.max(1, (int) (segmentSize * 0.1));
+
+    Assert.assertTrue(refillWatermark > 1);
+
+    // Make sure refill happens on the second page. At this point, there
+    // should be 3 calls made on the callable.
+    for (int i = 0; i < segmentSize * 2 - refillWatermark + 1; i++) {
+      underTest.getCurrentRow();
+      underTest.advance();
+    }
+
+    verify(callable, times(3))
+        .call(queryCaptor.capture(), any(ResponseObserver.class), any(ApiCallContext.class));
+  }
+
   /**
    * This test checks that the buffer will stop filling up once the byte limit is reached. It will
    * cancel the controller after reached the limit. This test completes one fill and contains one
@@ -747,18 +810,26 @@ public void run() {
    * @throws InterruptedException
    */
   @Test
-  public void testWrite() throws IOException, InterruptedException {
+  public void testWrite() throws IOException {
     doReturn(mockBatcher).when(mockBigtableDataClient).newBulkMutationBatcher(any());
+    SettableApiFuture<Void> fakeFuture = SettableApiFuture.create();
+    when(mockBatcher.closeAsync()).thenReturn(fakeFuture);
     ArgumentCaptor<RowMutationEntry> captor = ArgumentCaptor.forClass(RowMutationEntry.class);
     ApiFuture<Void> fakeResponse = SettableApiFuture.create();
-    when(mockBatcher.add(any(RowMutationEntry.class))).thenReturn(fakeResponse);
+    when(mockBatcher.add(any(RowMutationEntry.class)))
+        .thenAnswer(
+            invocation -> {
+              fakeFuture.set(null);
+              return fakeResponse;
+            });
 
     BigtableService.Writer underTest =
         new BigtableServiceImpl.BigtableWriterImpl(
             mockBigtableDataClient,
             bigtableDataSettings.getProjectId(),
             bigtableDataSettings.getInstanceId(),
-            TABLE_ID);
+            TABLE_ID,
+            Duration.millis(60000));
 
     ByteString key = ByteString.copyFromUtf8("key");
     Mutation mutation =
@@ -780,7 +851,6 @@ public void testWrite() throws IOException, InterruptedException {
     assertEquals(
         "Family", captor.getValue().toProto().getMutations(0).getSetCell().getFamilyName());
     underTest.close();
-    verify(mockBatcher, times(1)).flush();
   }
 
   private void verifyMetricWasSet(String method, String status, long count) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
index 3280e17998e4..ab6f5567dd9a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1Test.java
@@ -125,6 +125,7 @@
 @RunWith(Enclosed.class)
 public class DatastoreV1Test {
   private static final String PROJECT_ID = "testProject";
+  private static final String DATABASE_ID = "";
   private static final String NAMESPACE = "testNamespace";
   private static final String KIND = "testKind";
   private static final Query QUERY;
@@ -138,7 +139,7 @@ public class DatastoreV1Test {
     Query.Builder q = Query.newBuilder();
     q.addKindBuilder().setName(KIND);
     QUERY = q.build();
-    V_1_OPTIONS = V1Options.from(PROJECT_ID, NAMESPACE, null);
+    V_1_OPTIONS = V1Options.from(PROJECT_ID, StaticValueProvider.of(DATABASE_ID), NAMESPACE, null);
   }
 
   @Mock protected Datastore mockDatastore;
@@ -153,6 +154,9 @@ public void setUp() {
 
     when(mockDatastoreFactory.getDatastore(any(PipelineOptions.class), any(String.class), any()))
         .thenReturn(mockDatastore);
+    when(mockDatastoreFactory.getDatastore(
+            any(PipelineOptions.class), any(String.class), any(String.class), any()))
+        .thenReturn(mockDatastore);
     when(mockDatastoreFactory.getQuerySplitter()).thenReturn(mockQuerySplitter);
     // Setup the ProcessWideContainer for testing metrics are set.
     MetricsContainerImpl container = new MetricsContainerImpl(null);
@@ -167,10 +171,12 @@ public void testBuildRead() throws Exception {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withQuery(QUERY)
               .withNamespace(NAMESPACE);
       assertEquals(QUERY, read.getQuery());
       assertEquals(PROJECT_ID, read.getProjectId().get());
+      assertEquals(DATABASE_ID, read.getDatabaseId().get());
       assertEquals(NAMESPACE, read.getNamespace().get());
     }
 
@@ -180,11 +186,13 @@ public void testBuildReadWithReadTime() throws Exception {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withQuery(QUERY)
               .withReadTime(TIMESTAMP);
       assertEquals(TIMESTAMP, read.getReadTime());
       assertEquals(QUERY, read.getQuery());
       assertEquals(PROJECT_ID, read.getProjectId().get());
+      assertEquals(DATABASE_ID, read.getDatabaseId().get());
     }
 
     @Test
@@ -193,10 +201,12 @@ public void testBuildReadWithGqlQuery() throws Exception {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withLiteralGqlQuery(GQL_QUERY)
               .withNamespace(NAMESPACE);
       assertEquals(GQL_QUERY, read.getLiteralGqlQuery().get());
       assertEquals(PROJECT_ID, read.getProjectId().get());
+      assertEquals(DATABASE_ID, read.getDatabaseId().get());
       assertEquals(NAMESPACE, read.getNamespace().get());
     }
 
@@ -210,10 +220,12 @@ public void testBuildReadAlt() throws Exception {
               .withQuery(QUERY)
               .withNamespace(NAMESPACE)
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withLocalhost(LOCALHOST);
       assertEquals(TIMESTAMP, read.getReadTime());
       assertEquals(QUERY, read.getQuery());
       assertEquals(PROJECT_ID, read.getProjectId().get());
+      assertEquals(DATABASE_ID, read.getDatabaseId().get());
       assertEquals(NAMESPACE, read.getNamespace().get());
       assertEquals(LOCALHOST, read.getLocalhost());
     }
@@ -224,6 +236,7 @@ public void testReadValidationFailsQueryAndGqlQuery() throws Exception {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withLiteralGqlQuery(GQL_QUERY)
               .withQuery(QUERY);
 
@@ -257,6 +270,7 @@ public void testReadDisplayData() {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withQuery(QUERY)
               .withNamespace(NAMESPACE)
               .withReadTime(TIMESTAMP);
@@ -264,6 +278,7 @@ public void testReadDisplayData() {
       DisplayData displayData = DisplayData.from(read);
 
       assertThat(displayData, hasDisplayItem("projectId", PROJECT_ID));
+      assertThat(displayData, hasDisplayItem("databaseId", DATABASE_ID));
       assertThat(displayData, hasDisplayItem("query", QUERY.toString()));
       assertThat(displayData, hasDisplayItem("namespace", NAMESPACE));
       assertThat(displayData, hasDisplayItem("readTime", TIMESTAMP));
@@ -275,6 +290,7 @@ public void testReadDisplayDataWithGqlQuery() {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withLiteralGqlQuery(GQL_QUERY)
               .withNamespace(NAMESPACE)
               .withReadTime(TIMESTAMP);
@@ -295,6 +311,7 @@ public void testSourcePrimitiveDisplayData() {
           DatastoreIO.v1()
               .read()
               .withProjectId(PROJECT_ID)
+              .withDatabaseId(DATABASE_ID)
               .withQuery(Query.newBuilder().build())
               .withNumQuerySplits(numSplits);
 
@@ -313,29 +330,34 @@ public void testSourcePrimitiveDisplayData() {
 
     @Test
     public void testWriteDisplayData() {
-      Write write = DatastoreIO.v1().write().withProjectId(PROJECT_ID);
+      Write write = DatastoreIO.v1().write().withProjectId(PROJECT_ID).withDatabaseId(DATABASE_ID);
 
       DisplayData displayData = DisplayData.from(write);
 
       assertThat(displayData, hasDisplayItem("projectId", PROJECT_ID));
+      assertThat(displayData, hasDisplayItem("databaseId", DATABASE_ID));
     }
 
     @Test
     public void testDeleteEntityDisplayData() {
-      DeleteEntity deleteEntity = DatastoreIO.v1().deleteEntity().withProjectId(PROJECT_ID);
+      DeleteEntity deleteEntity =
+          DatastoreIO.v1().deleteEntity().withProjectId(PROJECT_ID).withDatabaseId(DATABASE_ID);
 
       DisplayData displayData = DisplayData.from(deleteEntity);
 
       assertThat(displayData, hasDisplayItem("projectId", PROJECT_ID));
+      assertThat(displayData, hasDisplayItem("databaseId", DATABASE_ID));
     }
 
     @Test
     public void testDeleteKeyDisplayData() {
-      DeleteKey deleteKey = DatastoreIO.v1().deleteKey().withProjectId(PROJECT_ID);
+      DeleteKey deleteKey =
+          DatastoreIO.v1().deleteKey().withProjectId(PROJECT_ID).withDatabaseId(DATABASE_ID);
 
       DisplayData displayData = DisplayData.from(deleteKey);
 
       assertThat(displayData, hasDisplayItem("projectId", PROJECT_ID));
+      assertThat(displayData, hasDisplayItem("databaseId", DATABASE_ID));
     }
 
     @Test
@@ -600,6 +622,8 @@ private void datastoreWriterFnTest(int numMutations) throws Exception {
       while (start < numMutations) {
         int end = Math.min(numMutations, start + DatastoreV1.DATASTORE_BATCH_UPDATE_ENTITIES_START);
         CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
+        commitRequest.setProjectId(PROJECT_ID);
+        commitRequest.setDatabaseId(DATABASE_ID);
         commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
         commitRequest.addAllMutations(mutations.subList(start, end));
         // Verify all the batch requests were made with the expected mutations.
@@ -644,6 +668,8 @@ public void testDatastoreWriterFnWithLargeEntities() throws Exception {
       while (start < mutations.size()) {
         int end = Math.min(mutations.size(), start + entitiesPerRpc);
         CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
+        commitRequest.setProjectId(PROJECT_ID);
+        commitRequest.setDatabaseId(DATABASE_ID);
         commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
         commitRequest.addAllMutations(mutations.subList(start, end));
         // Verify all the batch requests were made with the expected mutations.
@@ -681,12 +707,16 @@ public void testDatastoreWriterFnWithDuplicateEntities() throws Exception {
       CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
       commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
       commitRequest.addAllMutations(mutations.subList(0, 2));
+      commitRequest.setProjectId(PROJECT_ID);
+      commitRequest.setDatabaseId(DATABASE_ID);
       verify(mockDatastore, times(1)).commit(commitRequest.build());
 
       // second invocation has key [0, 2] because the second 0 triggered a flush batch
       commitRequest = CommitRequest.newBuilder();
       commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
       commitRequest.addAllMutations(mutations.subList(2, 4));
+      commitRequest.setProjectId(PROJECT_ID);
+      commitRequest.setDatabaseId(DATABASE_ID);
       verify(mockDatastore, times(1)).commit(commitRequest.build());
       verifyMetricWasSet("BatchDatastoreWrite", "ok", "", 2);
     }
@@ -841,17 +871,26 @@ public void testEstimatedSizeBytes() throws Exception {
       long timestamp = 1234L;
 
       RunQueryRequest latestTimestampRequest =
-          makeRequest(makeLatestTimestampQuery(NAMESPACE), NAMESPACE, readTime);
+          makeRequest(
+              PROJECT_ID, DATABASE_ID, makeLatestTimestampQuery(NAMESPACE), NAMESPACE, readTime);
       RunQueryResponse latestTimestampResponse = makeLatestTimestampResponse(timestamp);
       // Per Kind statistics request and response
       RunQueryRequest statRequest =
-          makeRequest(makeStatKindQuery(NAMESPACE, timestamp), NAMESPACE, readTime);
+          makeRequest(
+              PROJECT_ID,
+              DATABASE_ID,
+              makeStatKindQuery(NAMESPACE, timestamp),
+              NAMESPACE,
+              readTime);
       RunQueryResponse statResponse = makeStatKindResponse(entityBytes);
 
       when(mockDatastore.runQuery(latestTimestampRequest)).thenReturn(latestTimestampResponse);
       when(mockDatastore.runQuery(statRequest)).thenReturn(statResponse);
 
-      assertEquals(entityBytes, getEstimatedSizeBytes(mockDatastore, QUERY, NAMESPACE, readTime));
+      assertEquals(
+          entityBytes,
+          getEstimatedSizeBytes(
+              mockDatastore, PROJECT_ID, DATABASE_ID, QUERY, NAMESPACE, readTime));
       verify(mockDatastore, times(1)).runQuery(latestTimestampRequest);
       verify(mockDatastore, times(1)).runQuery(statRequest);
     }
@@ -917,12 +956,18 @@ public void testSplitQueryFnWithoutNumSplits() throws Exception {
       long timestamp = 1234L;
 
       RunQueryRequest latestTimestampRequest =
-          makeRequest(makeLatestTimestampQuery(NAMESPACE), NAMESPACE, readTime);
+          makeRequest(
+              PROJECT_ID, DATABASE_ID, makeLatestTimestampQuery(NAMESPACE), NAMESPACE, readTime);
       RunQueryResponse latestTimestampResponse = makeLatestTimestampResponse(timestamp);
 
       // Per Kind statistics request and response
       RunQueryRequest statRequest =
-          makeRequest(makeStatKindQuery(NAMESPACE, timestamp), NAMESPACE, readTime);
+          makeRequest(
+              PROJECT_ID,
+              DATABASE_ID,
+              makeStatKindQuery(NAMESPACE, timestamp),
+              NAMESPACE,
+              readTime);
       RunQueryResponse statResponse = makeStatKindResponse(entityBytes);
 
       when(mockDatastore.runQuery(latestTimestampRequest)).thenReturn(latestTimestampResponse);
@@ -1030,9 +1075,11 @@ public void testTranslateGqlQueryWithLimit() throws Exception {
       GqlQuery gqlQueryWithZeroLimit =
           GqlQuery.newBuilder().setQueryString(gqlWithZeroLimit).setAllowLiterals(true).build();
 
-      RunQueryRequest gqlRequest = makeRequest(gqlQuery, V_1_OPTIONS.getNamespace(), readTime);
+      RunQueryRequest gqlRequest =
+          makeRequest(PROJECT_ID, DATABASE_ID, gqlQuery, V_1_OPTIONS.getNamespace(), readTime);
       RunQueryRequest gqlRequestWithZeroLimit =
-          makeRequest(gqlQueryWithZeroLimit, V_1_OPTIONS.getNamespace(), readTime);
+          makeRequest(
+              PROJECT_ID, DATABASE_ID, gqlQueryWithZeroLimit, V_1_OPTIONS.getNamespace(), readTime);
       when(mockDatastore.runQuery(gqlRequestWithZeroLimit))
           .thenThrow(
               new DatastoreException(
@@ -1044,7 +1091,8 @@ public void testTranslateGqlQueryWithLimit() throws Exception {
       when(mockDatastore.runQuery(gqlRequest))
           .thenReturn(RunQueryResponse.newBuilder().setQuery(QUERY).build());
       assertEquals(
-          translateGqlQueryWithLimitCheck(gql, mockDatastore, V_1_OPTIONS.getNamespace(), readTime),
+          translateGqlQueryWithLimitCheck(
+              gql, mockDatastore, PROJECT_ID, DATABASE_ID, V_1_OPTIONS.getNamespace(), readTime),
           QUERY);
       verify(mockDatastore, times(1)).runQuery(gqlRequest);
       verify(mockDatastore, times(1)).runQuery(gqlRequestWithZeroLimit);
@@ -1058,11 +1106,18 @@ public void testTranslateGqlQueryWithNoLimit() throws Exception {
           GqlQuery.newBuilder().setQueryString(gqlWithZeroLimit).setAllowLiterals(true).build();
 
       RunQueryRequest gqlRequestWithZeroLimit =
-          makeRequest(gqlQueryWithZeroLimit, V_1_OPTIONS.getNamespace(), readTime);
+          makeRequest(
+              PROJECT_ID, DATABASE_ID, gqlQueryWithZeroLimit, V_1_OPTIONS.getNamespace(), readTime);
       when(mockDatastore.runQuery(gqlRequestWithZeroLimit))
           .thenReturn(RunQueryResponse.newBuilder().setQuery(QUERY).build());
       assertEquals(
-          translateGqlQueryWithLimitCheck(gql, mockDatastore, V_1_OPTIONS.getNamespace(), readTime),
+          translateGqlQueryWithLimitCheck(
+              gql,
+              mockDatastore,
+              V_1_OPTIONS.getProjectId(),
+              V_1_OPTIONS.getDatabaseId(),
+              V_1_OPTIONS.getNamespace(),
+              readTime),
           QUERY);
       verify(mockDatastore, times(1)).runQuery(gqlRequestWithZeroLimit);
     }
@@ -1074,13 +1129,20 @@ public void testTranslateGqlQueryWithException() throws Exception {
       GqlQuery gqlQueryWithZeroLimit =
           GqlQuery.newBuilder().setQueryString(gqlWithZeroLimit).setAllowLiterals(true).build();
       RunQueryRequest gqlRequestWithZeroLimit =
-          makeRequest(gqlQueryWithZeroLimit, V_1_OPTIONS.getNamespace(), readTime);
+          makeRequest(
+              PROJECT_ID, DATABASE_ID, gqlQueryWithZeroLimit, V_1_OPTIONS.getNamespace(), readTime);
       when(mockDatastore.runQuery(gqlRequestWithZeroLimit))
           .thenThrow(new RuntimeException("TestException"));
 
       thrown.expect(RuntimeException.class);
       thrown.expectMessage("TestException");
-      translateGqlQueryWithLimitCheck(gql, mockDatastore, V_1_OPTIONS.getNamespace(), readTime);
+      translateGqlQueryWithLimitCheck(
+          gql,
+          mockDatastore,
+          V_1_OPTIONS.getProjectId(),
+          V_1_OPTIONS.getDatabaseId(),
+          V_1_OPTIONS.getNamespace(),
+          readTime);
     }
   }
 
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java
index ea00821f3604..913e05c86129 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/SplitQueryFnIT.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.SplitQueryFn;
 import org.apache.beam.sdk.io.gcp.datastore.DatastoreV1.Read.V1Options;
+import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
 import org.apache.beam.sdk.transforms.DoFnTester;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.joda.time.Duration;
@@ -58,29 +59,32 @@ public class SplitQueryFnIT {
   @Test
   public void testSplitQueryFnWithLargeDataset() throws Exception {
     String projectId = "apache-beam-testing";
+    String databaseId = "";
     String kind = "sort_1G";
     String namespace = null;
     // Num splits is computed based on the entity_bytes size of the input_sort_1G kind reported by
     // Datastore stats.
     int expectedNumSplits = 32;
-    testSplitQueryFn(projectId, kind, namespace, expectedNumSplits, null);
-    testSplitQueryFn(projectId, kind, namespace, expectedNumSplits, readTime);
+    testSplitQueryFn(projectId, databaseId, kind, namespace, expectedNumSplits, null);
+    testSplitQueryFn(projectId, databaseId, kind, namespace, expectedNumSplits, readTime);
   }
 
   /** Tests {@link SplitQueryFn} to fallback to NUM_QUERY_SPLITS_MIN for a small dataset. */
   @Test
   public void testSplitQueryFnWithSmallDataset() throws Exception {
     String projectId = "apache-beam-testing";
+    String databaseId = "";
     String kind = "shakespeare";
     String namespace = null;
     int expectedNumSplits = NUM_QUERY_SPLITS_MIN;
-    testSplitQueryFn(projectId, kind, namespace, expectedNumSplits, null);
-    testSplitQueryFn(projectId, kind, namespace, expectedNumSplits, readTime);
+    testSplitQueryFn(projectId, databaseId, kind, namespace, expectedNumSplits, null);
+    testSplitQueryFn(projectId, databaseId, kind, namespace, expectedNumSplits, readTime);
   }
 
   /** A helper method to test {@link SplitQueryFn} to generate the expected number of splits. */
   private void testSplitQueryFn(
       String projectId,
+      String databaseId,
       String kind,
       @Nullable String namespace,
       int expectedNumSplits,
@@ -90,7 +94,10 @@ private void testSplitQueryFn(
     query.addKindBuilder().setName(kind);
 
     SplitQueryFn splitQueryFn =
-        new SplitQueryFn(V1Options.from(projectId, namespace, null), 0, readTime);
+        new SplitQueryFn(
+            V1Options.from(projectId, StaticValueProvider.of(databaseId), namespace, null),
+            0,
+            readTime);
     DoFnTester<Query, Query> doFnTester = DoFnTester.of(splitQueryFn);
 
     List<Query> queries = doFnTester.processBundle(query.build());
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
index 249cadd48866..c9530ba43cb0 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1ReadIT.java
@@ -48,6 +48,7 @@
 public class V1ReadIT {
   private V1TestOptions options;
   private String project;
+  private String database;
   private String ancestor;
   private final long numEntitiesBeforeReadTime = 600;
   private final long totalNumEntities = 1000;
@@ -58,9 +59,12 @@ public void setup() throws Exception {
     PipelineOptionsFactory.register(V1TestOptions.class);
     options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class);
     project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
+    // The default database.
+    database = "";
+
     ancestor = UUID.randomUUID().toString();
     // Create entities and write them to datastore
-    writeEntitiesToDatastore(options, project, ancestor, 0, numEntitiesBeforeReadTime);
+    writeEntitiesToDatastore(options, project, database, ancestor, 0, numEntitiesBeforeReadTime);
 
     Thread.sleep(1000);
     readTime = Instant.now();
@@ -68,12 +72,12 @@ public void setup() throws Exception {
 
     long moreEntitiesToWrite = totalNumEntities - numEntitiesBeforeReadTime;
     writeEntitiesToDatastore(
-        options, project, ancestor, numEntitiesBeforeReadTime, moreEntitiesToWrite);
+        options, project, database, ancestor, numEntitiesBeforeReadTime, moreEntitiesToWrite);
   }
 
   @After
   public void tearDown() throws Exception {
-    deleteAllEntities(options, project, ancestor);
+    deleteAllEntities(options, project, database, ancestor);
   }
 
   /**
@@ -93,6 +97,7 @@ public void testE2EV1Read() throws Exception {
         DatastoreIO.v1()
             .read()
             .withProjectId(project)
+            .withDatabaseId(database)
             .withQuery(query)
             .withNamespace(options.getNamespace());
 
@@ -108,6 +113,7 @@ public void testE2EV1Read() throws Exception {
         DatastoreIO.v1()
             .read()
             .withProjectId(project)
+            .withDatabaseId(database)
             .withQuery(query)
             .withNamespace(options.getNamespace())
             .withReadTime(readTime);
@@ -152,6 +158,7 @@ private void testE2EV1ReadWithGQLQuery(long limit) throws Exception {
         DatastoreIO.v1()
             .read()
             .withProjectId(project)
+            .withDatabaseId(database)
             .withLiteralGqlQuery(gqlQuery)
             .withNamespace(options.getNamespace());
 
@@ -167,6 +174,7 @@ private void testE2EV1ReadWithGQLQuery(long limit) throws Exception {
         DatastoreIO.v1()
             .read()
             .withProjectId(project)
+            .withDatabaseId(database)
             .withLiteralGqlQuery(gqlQuery)
             .withNamespace(options.getNamespace())
             .withReadTime(readTime);
@@ -181,11 +189,17 @@ private void testE2EV1ReadWithGQLQuery(long limit) throws Exception {
 
   // Creates entities and write them to datastore
   private static void writeEntitiesToDatastore(
-      V1TestOptions options, String project, String ancestor, long valueOffset, long numEntities)
+      V1TestOptions options,
+      String project,
+      String database,
+      String ancestor,
+      long valueOffset,
+      long numEntities)
       throws Exception {
-    Datastore datastore = getDatastore(options, project);
+    Datastore datastore = getDatastore(options, project, database);
     // Write test entities to datastore
-    V1TestWriter writer = new V1TestWriter(datastore, new UpsertMutationBuilder());
+    V1TestWriter writer =
+        new V1TestWriter(datastore, project, database, new UpsertMutationBuilder());
     Key ancestorKey = makeAncestorKey(options.getNamespace(), options.getKind(), ancestor);
 
     for (long i = 0; i < numEntities; i++) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
index e3057c4b90ca..0cb3ffac3222 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java
@@ -137,10 +137,12 @@ public void processElement(ProcessContext c) throws Exception {
   }
 
   /** Build a new datastore client. */
-  static Datastore getDatastore(PipelineOptions pipelineOptions, String projectId) {
+  static Datastore getDatastore(
+      PipelineOptions pipelineOptions, String projectId, String databaseId) {
     Credentials credential = pipelineOptions.as(GcpOptions.class).getGcpCredential();
     HttpRequestInitializer initializer;
     if (credential != null) {
+
       initializer =
           new ChainingHttpRequestInitializer(
               new HttpCredentialsAdapter(credential), new RetryHttpRequestInitializer());
@@ -149,14 +151,22 @@ static Datastore getDatastore(PipelineOptions pipelineOptions, String projectId)
     }
 
     DatastoreOptions.Builder builder =
-        new DatastoreOptions.Builder().projectId(projectId).initializer(initializer);
+        new DatastoreOptions.Builder()
+            .projectId(projectId)
+            .databaseId(databaseId)
+            .initializer(initializer);
 
     return DatastoreFactory.get().create(builder.build());
   }
 
   /** Build a datastore query request. */
-  private static RunQueryRequest makeRequest(Query query, @Nullable String namespace) {
-    RunQueryRequest.Builder requestBuilder = RunQueryRequest.newBuilder().setQuery(query);
+  private static RunQueryRequest makeRequest(
+      String projectId, String databaseId, Query query, @Nullable String namespace) {
+    RunQueryRequest.Builder requestBuilder =
+        RunQueryRequest.newBuilder()
+            .setQuery(query)
+            .setProjectId(projectId)
+            .setDatabaseId(databaseId);
     if (namespace != null) {
       requestBuilder.getPartitionIdBuilder().setNamespaceId(namespace);
     }
@@ -164,14 +174,16 @@ private static RunQueryRequest makeRequest(Query query, @Nullable String namespa
   }
 
   /** Delete all entities with the given ancestor. */
-  static void deleteAllEntities(V1TestOptions options, String project, String ancestor)
-      throws Exception {
-    Datastore datastore = getDatastore(options, project);
+  static void deleteAllEntities(
+      V1TestOptions options, String project, String database, String ancestor) throws Exception {
+    Datastore datastore = getDatastore(options, project, database);
     Query query =
         V1TestUtil.makeAncestorKindQuery(options.getKind(), options.getNamespace(), ancestor);
 
-    V1TestReader reader = new V1TestReader(datastore, query, options.getNamespace());
-    V1TestWriter writer = new V1TestWriter(datastore, new DeleteMutationBuilder());
+    V1TestReader reader =
+        new V1TestReader(datastore, project, database, query, options.getNamespace());
+    V1TestWriter writer =
+        new V1TestWriter(datastore, project, database, new DeleteMutationBuilder());
 
     long numEntities = 0;
     while (reader.advance()) {
@@ -185,14 +197,15 @@ static void deleteAllEntities(V1TestOptions options, String project, String ance
   }
 
   /** Returns the total number of entities for the given datastore. */
-  static long countEntities(V1TestOptions options, String project, String ancestor)
+  static long countEntities(V1TestOptions options, String project, String database, String ancestor)
       throws Exception {
     // Read from datastore.
-    Datastore datastore = V1TestUtil.getDatastore(options, project);
+    Datastore datastore = V1TestUtil.getDatastore(options, project, database);
     Query query =
         V1TestUtil.makeAncestorKindQuery(options.getKind(), options.getNamespace(), ancestor);
 
-    V1TestReader reader = new V1TestReader(datastore, query, options.getNamespace());
+    V1TestReader reader =
+        new V1TestReader(datastore, project, database, query, options.getNamespace());
 
     long numEntitiesRead = 0;
     while (reader.advance()) {
@@ -247,12 +260,17 @@ static boolean isValidKey(Key key) {
       return (lastElement.getId() != 0 || !lastElement.getName().isEmpty());
     }
 
+    private final String projectId;
+    private final String databaseId;
     private final Datastore datastore;
     private final MutationBuilder mutationBuilder;
     private final List<Entity> entities = new ArrayList<>();
 
-    V1TestWriter(Datastore datastore, MutationBuilder mutationBuilder) {
+    V1TestWriter(
+        Datastore datastore, String projectId, String databaseId, MutationBuilder mutationBuilder) {
       this.datastore = datastore;
+      this.projectId = projectId;
+      this.databaseId = databaseId;
       this.mutationBuilder = mutationBuilder;
     }
 
@@ -295,6 +313,8 @@ private void flushBatch() throws DatastoreException, IOException, InterruptedExc
             commitRequest.addMutations(mutationBuilder.apply(entity));
           }
           commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
+          commitRequest.setProjectId(projectId);
+          commitRequest.setDatabaseId(databaseId);
           datastore.commit(commitRequest.build());
           // Break if the commit threw no exception.
           break;
@@ -318,6 +338,8 @@ private void flushBatch() throws DatastoreException, IOException, InterruptedExc
   static class V1TestReader {
     private static final int QUERY_BATCH_LIMIT = 500;
     private final Datastore datastore;
+    private final String projectId;
+    private final String databaseId;
     private final Query query;
     private final @Nullable String namespace;
     private boolean moreResults;
@@ -326,8 +348,15 @@ static class V1TestReader {
     private QueryResultBatch currentBatch;
     private Entity currentEntity;
 
-    V1TestReader(Datastore datastore, Query query, @Nullable String namespace) {
+    V1TestReader(
+        Datastore datastore,
+        String projectId,
+        String databaseId,
+        Query query,
+        @Nullable String namespace) {
       this.datastore = datastore;
+      this.projectId = projectId;
+      this.databaseId = databaseId;
       this.query = query;
       this.namespace = namespace;
     }
@@ -362,7 +391,7 @@ private Iterator<EntityResult> getIteratorAndMoveCursor() throws DatastoreExcept
         query.setStartCursor(currentBatch.getEndCursor());
       }
 
-      RunQueryRequest request = makeRequest(query.build(), namespace);
+      RunQueryRequest request = makeRequest(projectId, databaseId, query.build(), namespace);
       RunQueryResponse response = datastore.runQuery(request);
 
       currentBatch = response.getBatch();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
index 4f794a03e58d..0062208630f6 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1WriteIT.java
@@ -56,6 +56,7 @@
 public class V1WriteIT {
   private V1TestOptions options;
   private String project;
+  private String database = "";
   private String ancestor;
   private final long numEntities = 1000;
 
@@ -83,12 +84,12 @@ public void testE2EV1Write() throws Exception {
             ParDo.of(
                 new V1TestUtil.CreateEntityFn(
                     options.getKind(), options.getNamespace(), ancestor, 0)))
-        .apply(DatastoreIO.v1().write().withProjectId(project));
+        .apply(DatastoreIO.v1().write().withProjectId(project).withDatabaseId(database));
 
     p.run();
 
     // Count number of entities written to datastore.
-    long numEntitiesWritten = countEntities(options, project, ancestor);
+    long numEntitiesWritten = countEntities(options, project, database, ancestor);
 
     assertEquals(numEntities, numEntitiesWritten);
   }
@@ -185,18 +186,18 @@ public void testE2EV1WriteWithLargeEntities() throws Exception {
             ParDo.of(
                 new V1TestUtil.CreateEntityFn(
                     options.getKind(), options.getNamespace(), ancestor, rawPropertySize)))
-        .apply(DatastoreIO.v1().write().withProjectId(project));
+        .apply(DatastoreIO.v1().write().withProjectId(project).withDatabaseId(database));
 
     p.run();
 
     // Count number of entities written to datastore.
-    long numEntitiesWritten = countEntities(options, project, ancestor);
+    long numEntitiesWritten = countEntities(options, project, database, ancestor);
 
     assertEquals(numLargeEntities, numEntitiesWritten);
   }
 
   @After
   public void tearDown() throws Exception {
-    deleteAllEntities(options, project, ancestor);
+    deleteAllEntities(options, project, database, ancestor);
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteDlqTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteDlqTest.java
index 04b98ed6008a..e23536d800e5 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteDlqTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteDlqTest.java
@@ -17,15 +17,26 @@
  */
 package org.apache.beam.sdk.io.gcp.pubsublite.internal;
 
+import static org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteReadSchemaTransformProvider.getRawBytesToRowFunction;
+import static org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteReadSchemaTransformProvider.getUuidFromMessage;
+
+import com.google.cloud.pubsublite.proto.AttributeValues;
 import com.google.cloud.pubsublite.proto.PubSubMessage;
 import com.google.cloud.pubsublite.proto.SequencedMessage;
 import com.google.protobuf.ByteString;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteIO;
 import org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteReadSchemaTransformProvider;
 import org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteReadSchemaTransformProvider.ErrorFn;
+import org.apache.beam.sdk.io.gcp.pubsublite.UuidDeduplicationOptions;
 import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling;
 import org.apache.beam.sdk.schemas.utils.JsonUtils;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
@@ -46,18 +57,131 @@
 @RunWith(JUnit4.class)
 public class PubsubLiteDlqTest {
 
-  private static final TupleTag<Row> OUTPUTTAG = PubsubLiteReadSchemaTransformProvider.OUTPUT_TAG;
-  private static final TupleTag<Row> ERRORTAG = PubsubLiteReadSchemaTransformProvider.ERROR_TAG;
+  private static final TupleTag<Row> OUTPUT_TAG = PubsubLiteReadSchemaTransformProvider.OUTPUT_TAG;
+  private static final TupleTag<Row> ERROR_TAG = PubsubLiteReadSchemaTransformProvider.ERROR_TAG;
 
-  private static final Schema BEAMSCHEMA =
+  private static final Schema BEAM_RAW_SCHEMA =
+      Schema.builder().addField("payload", Schema.FieldType.BYTES).build();
+  private static final Schema BEAM_SCHEMA =
       Schema.of(Schema.Field.of("name", Schema.FieldType.STRING));
-  private static final Schema ERRORSCHEMA = PubsubLiteReadSchemaTransformProvider.ERROR_SCHEMA;
+
+  private static final Schema BEAM_SCHEMA_ATTRIBUTES =
+      Schema.of(
+          Schema.Field.of("name", Schema.FieldType.STRING),
+          Schema.Field.of("key1", Schema.FieldType.STRING),
+          Schema.Field.of("key2", Schema.FieldType.STRING));
+
+  private static final Schema BEAM_SCHEMA_ATTRIBUTES_AND_MAP =
+      Schema.of(
+          Schema.Field.of("name", Schema.FieldType.STRING),
+          Schema.Field.of("key1", Schema.FieldType.STRING),
+          Schema.Field.of("key2", Schema.FieldType.STRING),
+          Schema.Field.of(
+              "attrs", Schema.FieldType.map(Schema.FieldType.STRING, Schema.FieldType.STRING)));
+
+  private static final Schema BEAM_SCHEMA_ATTRIBUTES_MAP =
+      Schema.of(
+          Schema.Field.of("name", Schema.FieldType.STRING),
+          Schema.Field.of(
+              "attrs", Schema.FieldType.map(Schema.FieldType.STRING, Schema.FieldType.STRING)));
+
+  private static final Map<String, String> STATIC_MAP;
+
+  static {
+    Map<String, String> tempMap = new HashMap<>();
+    tempMap.put("key1", "first_key");
+    tempMap.put("key2", "second_key");
+    STATIC_MAP = Collections.unmodifiableMap(tempMap);
+  }
+
+  private static final List<Row> RAW_ROWS;
+
+  static {
+    try {
+      RAW_ROWS =
+          Arrays.asList(
+              Row.withSchema(BEAM_RAW_SCHEMA)
+                  .withFieldValue("payload", "a".getBytes("UTF-8"))
+                  .build(),
+              Row.withSchema(BEAM_RAW_SCHEMA)
+                  .withFieldValue("payload", "b".getBytes("UTF-8"))
+                  .build(),
+              Row.withSchema(BEAM_RAW_SCHEMA)
+                  .withFieldValue("payload", "c".getBytes("UTF-8"))
+                  .build());
+    } catch (UnsupportedEncodingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private static final List<Row> ROWS_WITH_ATTRIBUTES =
+      Arrays.asList(
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES)
+              .withFieldValue("name", "a")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES)
+              .withFieldValue("name", "b")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES)
+              .withFieldValue("name", "c")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .build());
+  private static final List<Row> ROWS_WITH_ATTRIBUTES_MAP =
+      Arrays.asList(
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES_MAP)
+              .withFieldValue("name", "a")
+              .withFieldValue("attrs", STATIC_MAP)
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES_MAP)
+              .withFieldValue("name", "b")
+              .withFieldValue("attrs", STATIC_MAP)
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES_MAP)
+              .withFieldValue("name", "c")
+              .withFieldValue("attrs", STATIC_MAP)
+              .build());
+  private static final List<Row> ROWS_WITH_ATTRIBUTES_AND_MAP =
+      Arrays.asList(
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES_AND_MAP)
+              .withFieldValue("name", "a")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .withFieldValue("attrs", STATIC_MAP)
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES_AND_MAP)
+              .withFieldValue("name", "b")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .withFieldValue("attrs", STATIC_MAP)
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES_AND_MAP)
+              .withFieldValue("name", "c")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .withFieldValue("attrs", STATIC_MAP)
+              .build());
 
   private static final List<Row> ROWS =
       Arrays.asList(
-          Row.withSchema(BEAMSCHEMA).withFieldValue("name", "a").build(),
-          Row.withSchema(BEAMSCHEMA).withFieldValue("name", "b").build(),
-          Row.withSchema(BEAMSCHEMA).withFieldValue("name", "c").build());
+          Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(),
+          Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(),
+          Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build());
+
+  private static final Map<String, AttributeValues> ATTRIBUTE_VALUES_MAP = new HashMap<>();
+
+  static {
+    ATTRIBUTE_VALUES_MAP.put(
+        "key1",
+        AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8("first_key")).build());
+    ATTRIBUTE_VALUES_MAP.put(
+        "key2",
+        AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8("second_key")).build());
+  }
 
   private static final List<SequencedMessage> MESSAGES =
       Arrays.asList(
@@ -65,18 +189,45 @@ public class PubsubLiteDlqTest {
               .setMessage(
                   PubSubMessage.newBuilder()
                       .setData(ByteString.copyFromUtf8("{\"name\":\"a\"}"))
+                      .putAllAttributes(ATTRIBUTE_VALUES_MAP)
                       .build())
               .build(),
           SequencedMessage.newBuilder()
               .setMessage(
                   PubSubMessage.newBuilder()
                       .setData(ByteString.copyFromUtf8("{\"name\":\"b\"}"))
+                      .putAllAttributes(ATTRIBUTE_VALUES_MAP)
                       .build())
               .build(),
           SequencedMessage.newBuilder()
               .setMessage(
                   PubSubMessage.newBuilder()
                       .setData(ByteString.copyFromUtf8("{\"name\":\"c\"}"))
+                      .putAllAttributes(ATTRIBUTE_VALUES_MAP)
+                      .build())
+              .build());
+
+  private static final List<SequencedMessage> RAW_MESSAGES =
+      Arrays.asList(
+          SequencedMessage.newBuilder()
+              .setMessage(
+                  PubSubMessage.newBuilder()
+                      .setData(ByteString.copyFromUtf8("a"))
+                      .putAllAttributes(ATTRIBUTE_VALUES_MAP)
+                      .build())
+              .build(),
+          SequencedMessage.newBuilder()
+              .setMessage(
+                  PubSubMessage.newBuilder()
+                      .setData(ByteString.copyFromUtf8("b"))
+                      .putAllAttributes(ATTRIBUTE_VALUES_MAP)
+                      .build())
+              .build(),
+          SequencedMessage.newBuilder()
+              .setMessage(
+                  PubSubMessage.newBuilder()
+                      .setData(ByteString.copyFromUtf8("c"))
+                      .putAllAttributes(ATTRIBUTE_VALUES_MAP)
                       .build())
               .build());
 
@@ -102,40 +253,222 @@ public class PubsubLiteDlqTest {
               .build());
 
   final SerializableFunction<byte[], Row> valueMapper =
-      JsonUtils.getJsonBytesToRowFunction(BEAMSCHEMA);
+      JsonUtils.getJsonBytesToRowFunction(BEAM_SCHEMA);
 
   @Rule public transient TestPipeline p = TestPipeline.create();
 
   @Test
-  public void testPubsubLiteErrorFnSuccess() throws Exception {
+  public void testPubsubLiteErrorFnSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
     PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGES));
     PCollectionTuple output =
         input.apply(
-            ParDo.of(new ErrorFn("Read-Error-Counter", valueMapper))
-                .withOutputTags(OUTPUTTAG, TupleTagList.of(ERRORTAG)));
+            ParDo.of(new ErrorFn("Read-Error-Counter", valueMapper, errorSchema, Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
 
-    output.get(OUTPUTTAG).setRowSchema(BEAMSCHEMA);
-    output.get(ERRORTAG).setRowSchema(ERRORSCHEMA);
+    output.get(OUTPUT_TAG).setRowSchema(BEAM_SCHEMA);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
 
-    PAssert.that(output.get(OUTPUTTAG)).containsInAnyOrder(ROWS);
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(ROWS);
     p.run().waitUntilFinish();
   }
 
   @Test
-  public void testPubsubLiteErrorFnFailure() throws Exception {
+  public void testPubsubLiteErrorFnFailure() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
     PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGESWITHERROR));
     PCollectionTuple output =
         input.apply(
-            ParDo.of(new ErrorFn("Read-Error-Counter", valueMapper))
-                .withOutputTags(OUTPUTTAG, TupleTagList.of(ERRORTAG)));
+            ParDo.of(new ErrorFn("Read-Error-Counter", valueMapper, errorSchema, Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(OUTPUT_TAG).setRowSchema(BEAM_SCHEMA);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PCollection<Long> count = output.get(ERROR_TAG).apply("error_count", Count.globally());
+
+    PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L));
+
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnRawSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+
+    List<String> attributes = new ArrayList<>();
+    String attributesMap = "";
+    Schema beamAttributeSchema =
+        PubsubLiteReadSchemaTransformProvider.buildSchemaWithAttributes(
+            BEAM_RAW_SCHEMA, attributes, attributesMap);
+    SerializableFunction<byte[], Row> rawValueMapper = getRawBytesToRowFunction(BEAM_RAW_SCHEMA);
+    PCollection<SequencedMessage> input = p.apply(Create.of(RAW_MESSAGES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(new ErrorFn("Read-Error-Counter", rawValueMapper, errorSchema, Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(RAW_ROWS);
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnWithAttributesSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    List<String> attributes = new ArrayList<>();
+    attributes.add("key1");
+    attributes.add("key2");
+    String attributeMap = "";
+    Schema beamAttributeSchema =
+        PubsubLiteReadSchemaTransformProvider.buildSchemaWithAttributes(
+            BEAM_SCHEMA, attributes, attributeMap);
+
+    PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(
+                    new ErrorFn(
+                        "Read-Error-Counter",
+                        valueMapper,
+                        errorSchema,
+                        attributes,
+                        attributeMap,
+                        beamAttributeSchema,
+                        Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
 
-    output.get(OUTPUTTAG).setRowSchema(BEAMSCHEMA);
-    output.get(ERRORTAG).setRowSchema(ERRORSCHEMA);
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
 
-    PCollection<Long> count = output.get(ERRORTAG).apply("error_count", Count.globally());
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(ROWS_WITH_ATTRIBUTES);
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnWithAttributeMapSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    // empty list of attributes
+    List<String> attributes = new ArrayList<>();
+    String attributeMap = "attrs";
+    Schema beamAttributeSchema =
+        PubsubLiteReadSchemaTransformProvider.buildSchemaWithAttributes(
+            BEAM_SCHEMA, attributes, attributeMap);
+
+    PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(
+                    new ErrorFn(
+                        "Read-Error-Counter",
+                        valueMapper,
+                        errorSchema,
+                        attributes,
+                        attributeMap,
+                        beamAttributeSchema,
+                        Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(ROWS_WITH_ATTRIBUTES_MAP);
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnWithAttributesAndAttributeMapSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    List<String> attributes = new ArrayList<>();
+    attributes.add("key1");
+    attributes.add("key2");
+    String attributeMap = "attrs";
+    Schema beamAttributeSchema =
+        PubsubLiteReadSchemaTransformProvider.buildSchemaWithAttributes(
+            BEAM_SCHEMA, attributes, attributeMap);
+
+    PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(
+                    new ErrorFn(
+                        "Read-Error-Counter",
+                        valueMapper,
+                        errorSchema,
+                        attributes,
+                        attributeMap,
+                        beamAttributeSchema,
+                        Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(ROWS_WITH_ATTRIBUTES_AND_MAP);
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnWithAttributesFailure() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    List<String> attributes = new ArrayList<>();
+    attributes.add("randomKey1");
+    attributes.add("randomKey2");
+    String attributeMap = "";
+    Schema beamAttributeSchema =
+        PubsubLiteReadSchemaTransformProvider.buildSchemaWithAttributes(
+            BEAM_SCHEMA, attributes, attributeMap);
+
+    PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(
+                    new ErrorFn(
+                        "Read-Error-Counter",
+                        valueMapper,
+                        errorSchema,
+                        attributes,
+                        attributeMap,
+                        beamAttributeSchema,
+                        Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(OUTPUT_TAG).setRowSchema(beamAttributeSchema);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PCollection<Long> count = output.get(ERROR_TAG).apply("error_count", Count.globally());
 
     PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L));
 
     p.run().waitUntilFinish();
   }
+
+  @Test
+  public void testPubsubLiteErrorFnWithDedupingSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+
+    PCollection<SequencedMessage> input = p.apply(Create.of(MESSAGES));
+    UuidDeduplicationOptions.Builder uuidExtractor =
+        UuidDeduplicationOptions.newBuilder().setUuidExtractor(getUuidFromMessage("key1"));
+    PCollectionTuple output =
+        input
+            .apply(PubsubLiteIO.deduplicate(uuidExtractor.build()))
+            .apply(
+                ParDo.of(new ErrorFn("Read-Error-Counter", valueMapper, errorSchema, Boolean.TRUE))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(OUTPUT_TAG).setRowSchema(BEAM_SCHEMA);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PCollection<Long> count = output.get(OUTPUT_TAG).apply("error_count", Count.globally());
+
+    // We are deduping so we should only have 1 value
+    PAssert.that(count).containsInAnyOrder(Collections.singletonList(1L));
+
+    p.run().waitUntilFinish();
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteWriteDlqTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteWriteDlqTest.java
index d42eb249b27e..eac0e4264c1a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteWriteDlqTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/internal/PubsubLiteWriteDlqTest.java
@@ -17,16 +17,24 @@
  */
 package org.apache.beam.sdk.io.gcp.pubsublite.internal;
 
+import com.google.cloud.pubsublite.proto.AttributeValues;
 import com.google.cloud.pubsublite.proto.PubSubMessage;
 import com.google.protobuf.ByteString;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteWriteSchemaTransformProvider;
 import org.apache.beam.sdk.io.gcp.pubsublite.PubsubLiteWriteSchemaTransformProvider.ErrorCounterFn;
 import org.apache.beam.sdk.schemas.Schema;
+import org.apache.beam.sdk.schemas.transforms.providers.ErrorHandling;
 import org.apache.beam.sdk.schemas.utils.JsonUtils;
 import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SerializableFunction;
@@ -47,38 +55,188 @@ public class PubsubLiteWriteDlqTest {
       PubsubLiteWriteSchemaTransformProvider.OUTPUT_TAG;
   private static final TupleTag<Row> ERROR_TAG = PubsubLiteWriteSchemaTransformProvider.ERROR_TAG;
 
-  private static final Schema BEAMSCHEMA =
+  private static final Schema BEAM_SCHEMA =
       Schema.of(Schema.Field.of("name", Schema.FieldType.STRING));
-  private static final Schema ERRORSCHEMA = PubsubLiteWriteSchemaTransformProvider.ERROR_SCHEMA;
+
+  private static final Schema BEAM_RAW_SCHEMA =
+      Schema.of(Schema.Field.of("payload", Schema.FieldType.BYTES));
+
+  private static final Schema BEAM_SCHEMA_ATTRIBUTES =
+      Schema.of(
+          Schema.Field.of("name", Schema.FieldType.STRING),
+          Schema.Field.of("key1", Schema.FieldType.STRING),
+          Schema.Field.of("key2", Schema.FieldType.STRING));
+
+  private static final List<Row> RAW_ROWS;
+
+  static {
+    try {
+      RAW_ROWS =
+          Arrays.asList(
+              Row.withSchema(BEAM_RAW_SCHEMA)
+                  .withFieldValue("payload", "a".getBytes("UTF8"))
+                  .build(),
+              Row.withSchema(BEAM_RAW_SCHEMA)
+                  .withFieldValue("payload", "b".getBytes("UTF8"))
+                  .build(),
+              Row.withSchema(BEAM_RAW_SCHEMA)
+                  .withFieldValue("payload", "c".getBytes("UTF8"))
+                  .build());
+    } catch (UnsupportedEncodingException e) {
+      throw new RuntimeException(e);
+    }
+  }
 
   private static final List<Row> ROWS =
       Arrays.asList(
-          Row.withSchema(BEAMSCHEMA).withFieldValue("name", "a").build(),
-          Row.withSchema(BEAMSCHEMA).withFieldValue("name", "b").build(),
-          Row.withSchema(BEAMSCHEMA).withFieldValue("name", "c").build());
+          Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "a").build(),
+          Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "b").build(),
+          Row.withSchema(BEAM_SCHEMA).withFieldValue("name", "c").build());
+
+  private static final List<Row> ROWSATTRIBUTES =
+      Arrays.asList(
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES)
+              .withFieldValue("name", "a")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES)
+              .withFieldValue("name", "b")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .build(),
+          Row.withSchema(BEAM_SCHEMA_ATTRIBUTES)
+              .withFieldValue("name", "c")
+              .withFieldValue("key1", "first_key")
+              .withFieldValue("key2", "second_key")
+              .build());
+
+  private static final Map<String, AttributeValues> ATTRIBUTE_VALUES_MAP = new HashMap<>();
+
+  static {
+    ATTRIBUTE_VALUES_MAP.put(
+        "key1",
+        AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8("first_key")).build());
+    ATTRIBUTE_VALUES_MAP.put(
+        "key2",
+        AttributeValues.newBuilder().addValues(ByteString.copyFromUtf8("second_key")).build());
+  }
+
+  private static final List<PubSubMessage> MESSAGES_RAW =
+      Arrays.asList(
+          PubSubMessage.newBuilder().setData(ByteString.copyFromUtf8("a")).build(),
+          PubSubMessage.newBuilder().setData(ByteString.copyFromUtf8("b")).build(),
+          PubSubMessage.newBuilder().setData(ByteString.copyFromUtf8("c")).build());
 
   private static final List<PubSubMessage> MESSAGES =
       Arrays.asList(
           PubSubMessage.newBuilder().setData(ByteString.copyFromUtf8("{\"name\":\"a\"}")).build(),
           PubSubMessage.newBuilder().setData(ByteString.copyFromUtf8("{\"name\":\"b\"}")).build(),
           PubSubMessage.newBuilder().setData(ByteString.copyFromUtf8("{\"name\":\"c\"}")).build());
+  private static final List<PubSubMessage> MESSAGES_WITH_ATTRIBUTES =
+      Arrays.asList(
+          PubSubMessage.newBuilder()
+              .setData(ByteString.copyFromUtf8("{\"name\":\"a\"}"))
+              .putAllAttributes(ATTRIBUTE_VALUES_MAP)
+              .build(),
+          PubSubMessage.newBuilder()
+              .setData(ByteString.copyFromUtf8("{\"name\":\"b\"}"))
+              .putAllAttributes(ATTRIBUTE_VALUES_MAP)
+              .build(),
+          PubSubMessage.newBuilder()
+              .setData(ByteString.copyFromUtf8("{\"name\":\"c\"}"))
+              .putAllAttributes(ATTRIBUTE_VALUES_MAP)
+              .build());
 
   final SerializableFunction<Row, byte[]> valueMapper =
-      JsonUtils.getRowToJsonBytesFunction(BEAMSCHEMA);
+      JsonUtils.getRowToJsonBytesFunction(BEAM_SCHEMA);
+
+  final SerializableFunction<Row, byte[]> valueMapperRaw =
+      PubsubLiteWriteSchemaTransformProvider.getRowToRawBytesFunction("payload");
 
   @Rule public transient TestPipeline p = TestPipeline.create();
 
   @Test
-  public void testPubsubLiteErrorFnSuccess() throws Exception {
+  public void testPubsubLiteErrorFnSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
     PCollection<Row> input = p.apply(Create.of(ROWS));
     PCollectionTuple output =
         input.apply(
-            ParDo.of(new ErrorCounterFn("ErrorCounter", valueMapper))
+            ParDo.of(new ErrorCounterFn("ErrorCounter", valueMapper, errorSchema, Boolean.TRUE))
                 .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
 
-    output.get(ERROR_TAG).setRowSchema(ERRORSCHEMA);
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
 
     PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(MESSAGES);
     p.run().waitUntilFinish();
   }
+
+  @Test
+  public void testPubsubLiteErrorFnSuccessRawEvents() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    PCollection<Row> input = p.apply(Create.of(RAW_ROWS));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(new ErrorCounterFn("ErrorCounter", valueMapperRaw, errorSchema, Boolean.TRUE))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(MESSAGES_RAW);
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnSuccessWithAttributes() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    List<String> attributes = new ArrayList<>();
+    attributes.add("key1");
+    attributes.add("key2");
+    Schema schema =
+        PubsubLiteWriteSchemaTransformProvider.getSchemaWithoutAttributes(
+            BEAM_SCHEMA_ATTRIBUTES, attributes);
+    PCollection<Row> input = p.apply(Create.of(ROWSATTRIBUTES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(
+                    new ErrorCounterFn(
+                        "ErrorCounter", valueMapper, errorSchema, Boolean.TRUE, attributes, schema))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PAssert.that(output.get(OUTPUT_TAG)).containsInAnyOrder(MESSAGES_WITH_ATTRIBUTES);
+    p.run().waitUntilFinish();
+  }
+
+  @Test
+  public void testPubsubLiteErrorFnSuccessWithAttributesAndDedupingSuccess() {
+    Schema errorSchema = ErrorHandling.errorSchemaBytes();
+    List<String> attributes = new ArrayList<>();
+    attributes.add("key1");
+    attributes.add("key2");
+    Schema schema =
+        PubsubLiteWriteSchemaTransformProvider.getSchemaWithoutAttributes(
+            BEAM_SCHEMA_ATTRIBUTES, attributes);
+    PCollection<Row> input = p.apply(Create.of(ROWSATTRIBUTES));
+    PCollectionTuple output =
+        input.apply(
+            ParDo.of(
+                    new ErrorCounterFn(
+                        "ErrorCounter", valueMapper, errorSchema, Boolean.TRUE, attributes, schema))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(ERROR_TAG)));
+
+    output.get(ERROR_TAG).setRowSchema(errorSchema);
+
+    PCollection<Long> count =
+        output
+            .get(OUTPUT_TAG)
+            .apply(
+                ParDo.of(
+                    new PubsubLiteWriteSchemaTransformProvider.SetUuidFromPubSubMessage.SetUuidFn(
+                        "unique_key")))
+            .apply("error_count", Count.globally());
+    PAssert.that(count).containsInAnyOrder(Collections.singletonList(3L));
+    p.run().waitUntilFinish();
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java
index 94f632e7daa1..67cf729c417f 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsKmsKeyIT.java
@@ -70,7 +70,8 @@ public void testGcsWriteWithKmsKey() {
     TestPipelineOptions options =
         TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class);
     assertNotNull(options.getTempRoot());
-    options.setTempLocation(options.getTempRoot() + "/testGcsWriteWithKmsKey");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "testGcsWriteWithKmsKey").toString());
     GcsOptions gcsOptions = options.as(GcsOptions.class);
 
     ResourceId filenamePrefix =
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java
index 336bcd768f7a..5f04c973adee 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/storage/GcsMatchIT.java
@@ -144,7 +144,8 @@ public Void apply(Iterable<Metadata> input) {
   public static void setUp() throws Exception {
     options = TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class);
     assertNotNull(options.getTempRoot());
-    options.setTempLocation(options.getTempRoot() + "/GcsMatchIT");
+    options.setTempLocation(
+        FileSystems.matchNewDirectory(options.getTempRoot(), "GcsMatchIT").toString());
     GcsOptions gcsOptions = options.as(GcsOptions.class);
     String dstFolderName =
         gcsOptions.getGcpTempLocation()
diff --git a/sdks/java/io/kafka/kafka-01103/build.gradle b/sdks/java/io/kafka/kafka-01103/build.gradle
index a0fa372397a2..3a74bf04ef22 100644
--- a/sdks/java/io/kafka/kafka-01103/build.gradle
+++ b/sdks/java/io/kafka/kafka-01103/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="0.11.0.3"
     undelimited="01103"
+    sdfCompatible=false
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-100/build.gradle b/sdks/java/io/kafka/kafka-100/build.gradle
index 15ce8c0deeff..bd5fa67b1cfc 100644
--- a/sdks/java/io/kafka/kafka-100/build.gradle
+++ b/sdks/java/io/kafka/kafka-100/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="1.0.0"
     undelimited="100"
+    sdfCompatible=false
 }
 
-apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
+apply from: "../kafka-integration-test.gradle"
diff --git a/sdks/java/io/kafka/kafka-111/build.gradle b/sdks/java/io/kafka/kafka-111/build.gradle
index fee4c382ed41..c2b0c8f82827 100644
--- a/sdks/java/io/kafka/kafka-111/build.gradle
+++ b/sdks/java/io/kafka/kafka-111/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="1.1.1"
     undelimited="111"
+    sdfCompatible=false
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-201/build.gradle b/sdks/java/io/kafka/kafka-201/build.gradle
index d395d0aa6269..a26ca4ac19cf 100644
--- a/sdks/java/io/kafka/kafka-201/build.gradle
+++ b/sdks/java/io/kafka/kafka-201/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="2.0.1"
     undelimited="201"
+    sdfCompatible=true
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-211/build.gradle b/sdks/java/io/kafka/kafka-211/build.gradle
index 4de07193b5a2..433d6c93f361 100644
--- a/sdks/java/io/kafka/kafka-211/build.gradle
+++ b/sdks/java/io/kafka/kafka-211/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="2.1.1"
     undelimited="211"
+    sdfCompatible=true
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-222/build.gradle b/sdks/java/io/kafka/kafka-222/build.gradle
index 57de58e81895..0f037e742968 100644
--- a/sdks/java/io/kafka/kafka-222/build.gradle
+++ b/sdks/java/io/kafka/kafka-222/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="2.2.2"
     undelimited="222"
+    sdfCompatible=true
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-231/build.gradle b/sdks/java/io/kafka/kafka-231/build.gradle
index 3682791c5b67..712158dcd3ae 100644
--- a/sdks/java/io/kafka/kafka-231/build.gradle
+++ b/sdks/java/io/kafka/kafka-231/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="2.3.1"
     undelimited="231"
+    sdfCompatible=true
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-241/build.gradle b/sdks/java/io/kafka/kafka-241/build.gradle
index 358c95aeb2fe..c0ac7df674b5 100644
--- a/sdks/java/io/kafka/kafka-241/build.gradle
+++ b/sdks/java/io/kafka/kafka-241/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="2.4.1"
     undelimited="241"
+    sdfCompatible=true
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-251/build.gradle b/sdks/java/io/kafka/kafka-251/build.gradle
index f291ecccc36b..4de9f97a738a 100644
--- a/sdks/java/io/kafka/kafka-251/build.gradle
+++ b/sdks/java/io/kafka/kafka-251/build.gradle
@@ -18,6 +18,7 @@
 project.ext {
     delimited="2.5.1"
     undelimited="251"
+    sdfCompatible=true
 }
 
 apply from: "../kafka-integration-test.gradle"
\ No newline at end of file
diff --git a/sdks/java/io/kafka/kafka-integration-test.gradle b/sdks/java/io/kafka/kafka-integration-test.gradle
index 778f8a3c456c..1aeb0c97f93b 100644
--- a/sdks/java/io/kafka/kafka-integration-test.gradle
+++ b/sdks/java/io/kafka/kafka-integration-test.gradle
@@ -39,4 +39,4 @@ dependencies {
 
 configurations.create("kafkaVersion$undelimited")
 
-tasks.register("kafkaVersion${undelimited}BatchIT",KafkaTestUtilities.KafkaBatchIT, project.ext.delimited, project.ext.undelimited, false, configurations, project)
\ No newline at end of file
+tasks.register("kafkaVersion${undelimited}BatchIT",KafkaTestUtilities.KafkaBatchIT, project.ext.delimited, project.ext.undelimited, project.ext.sdfCompatible, configurations, project)
\ No newline at end of file
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
index 7e4fc55c6ce9..8fd0c34cfa90 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIO.java
@@ -81,6 +81,11 @@
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.SimpleFunction;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.DefaultErrorHandler;
 import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimator;
 import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators.Manual;
 import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators.MonotonicallyIncreasing;
@@ -89,9 +94,11 @@
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
 import org.apache.beam.sdk.values.PDone;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Joiner;
@@ -167,6 +174,10 @@
  *      // signal.
  *      .withCheckStopReadingFn(new SerializedFunction<TopicPartition, Boolean>() {})
  *
+ *      //If you would like to send messages that fail to be parsed from Kafka to an alternate sink,
+ *      //use the error handler pattern as defined in {@link ErrorHandler}
+ *      .withBadRecordErrorHandler(errorHandler)
+ *
  *      // finally, if you don't need Kafka metadata, you can drop it.g
  *      .withoutMetadata() // PCollection<KV<Long, String>>
  *   )
@@ -469,6 +480,11 @@
  *      // or you can also set a custom timestamp with a function.
  *      .withPublishTimestampFunction((elem, elemTs) -> ...)
  *
+ *      // Optionally, records that fail to serialize can be sent to an error handler
+ *      // See {@link ErrorHandler} for details of for details of configuring a bad record error
+ *      // handler
+ *      .withBadRecordErrorHandler(errorHandler)
+ *
  *      // Optionally enable exactly-once sink (on supported runners). See JavaDoc for withEOS().
  *      .withEOS(20, "eos-sink-group-id");
  *   );
@@ -592,13 +608,7 @@ public static <K, V> ReadSourceDescriptors<K, V> readSourceDescriptors() {
    */
   public static <K, V> Write<K, V> write() {
     return new AutoValue_KafkaIO_Write.Builder<K, V>()
-        .setWriteRecordsTransform(
-            new AutoValue_KafkaIO_WriteRecords.Builder<K, V>()
-                .setProducerConfig(WriteRecords.DEFAULT_PRODUCER_PROPERTIES)
-                .setEOS(false)
-                .setNumShards(0)
-                .setConsumerFactoryFn(KafkaIOUtils.KAFKA_CONSUMER_FACTORY_FN)
-                .build())
+        .setWriteRecordsTransform(writeRecords())
         .build();
   }
 
@@ -613,6 +623,8 @@ public static <K, V> WriteRecords<K, V> writeRecords() {
         .setEOS(false)
         .setNumShards(0)
         .setConsumerFactoryFn(KafkaIOUtils.KAFKA_CONSUMER_FACTORY_FN)
+        .setBadRecordRouter(BadRecordRouter.THROWING_ROUTER)
+        .setBadRecordErrorHandler(new DefaultErrorHandler<>())
         .build();
   }
 
@@ -691,6 +703,9 @@ public abstract static class Read<K, V>
     @Pure
     public abstract @Nullable CheckStopReadingFn getCheckStopReadingFn();
 
+    @Pure
+    public abstract @Nullable ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
     abstract Builder<K, V> toBuilder();
 
     @AutoValue.Builder
@@ -739,6 +754,9 @@ abstract Builder<K, V> setValueDeserializerProvider(
 
       abstract Builder<K, V> setCheckStopReadingFn(@Nullable CheckStopReadingFn checkStopReadingFn);
 
+      abstract Builder<K, V> setBadRecordErrorHandler(
+          @Nullable ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
       Builder<K, V> setCheckStopReadingFn(
           @Nullable SerializableFunction<TopicPartition, Boolean> checkStopReadingFn) {
         return setCheckStopReadingFn(CheckStopReadingFnWrapper.of(checkStopReadingFn));
@@ -1312,6 +1330,10 @@ public Read<K, V> withCheckStopReadingFn(
           .build();
     }
 
+    public Read<K, V> withBadRecordErrorHandler(ErrorHandler<BadRecord, ?> badRecordErrorHandler) {
+      return toBuilder().setBadRecordErrorHandler(badRecordErrorHandler).build();
+    }
+
     /** Returns a {@link PTransform} for PCollection of {@link KV}, dropping Kafka metatdata. */
     public PTransform<PBegin, PCollection<KV<K, V>>> withoutMetadata() {
       return new TypedWithoutMetadata<>(this);
@@ -1529,6 +1551,11 @@ static class ReadFromKafkaViaUnbounded<K, V> extends AbstractReadFromKafka<K, V>
 
       @Override
       public PCollection<KafkaRecord<K, V>> expand(PBegin input) {
+        if (kafkaRead.getBadRecordErrorHandler() != null) {
+          LOG.warn(
+              "The Legacy implementation of Kafka Read does not support writing malformed"
+                  + "messages to an error handler. Use the SDF implementation instead.");
+        }
         // Handles unbounded source to bounded conversion if maxNumRecords or maxReadTime is set.
         Unbounded<KafkaRecord<K, V>> unbounded =
             org.apache.beam.sdk.io.Read.from(
@@ -1576,6 +1603,10 @@ public PCollection<KafkaRecord<K, V>> expand(PBegin input) {
         if (kafkaRead.getStopReadTime() != null) {
           readTransform = readTransform.withBounded();
         }
+        if (kafkaRead.getBadRecordErrorHandler() != null) {
+          readTransform =
+              readTransform.withBadRecordErrorHandler(kafkaRead.getBadRecordErrorHandler());
+        }
         PCollection<KafkaSourceDescriptor> output;
         if (kafkaRead.isDynamicRead()) {
           Set<String> topics = new HashSet<>();
@@ -1956,6 +1987,8 @@ public void populateDisplayData(DisplayData.Builder builder) {
   public abstract static class ReadSourceDescriptors<K, V>
       extends PTransform<PCollection<KafkaSourceDescriptor>, PCollection<KafkaRecord<K, V>>> {
 
+    private final TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> records = new TupleTag<>();
+
     private static final Logger LOG = LoggerFactory.getLogger(ReadSourceDescriptors.class);
 
     @Pure
@@ -1997,6 +2030,12 @@ public abstract static class ReadSourceDescriptors<K, V>
     @Pure
     abstract @Nullable TimestampPolicyFactory<K, V> getTimestampPolicyFactory();
 
+    @Pure
+    abstract BadRecordRouter getBadRecordRouter();
+
+    @Pure
+    abstract ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
     abstract boolean isBounded();
 
     abstract ReadSourceDescriptors.Builder<K, V> toBuilder();
@@ -2041,6 +2080,12 @@ abstract ReadSourceDescriptors.Builder<K, V> setCommitOffsetEnabled(
       abstract ReadSourceDescriptors.Builder<K, V> setTimestampPolicyFactory(
           TimestampPolicyFactory<K, V> policy);
 
+      abstract ReadSourceDescriptors.Builder<K, V> setBadRecordRouter(
+          BadRecordRouter badRecordRouter);
+
+      abstract ReadSourceDescriptors.Builder<K, V> setBadRecordErrorHandler(
+          ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
       abstract ReadSourceDescriptors.Builder<K, V> setBounded(boolean bounded);
 
       abstract ReadSourceDescriptors<K, V> build();
@@ -2052,6 +2097,8 @@ public static <K, V> ReadSourceDescriptors<K, V> read() {
           .setConsumerConfig(KafkaIOUtils.DEFAULT_CONSUMER_PROPERTIES)
           .setCommitOffsetEnabled(false)
           .setBounded(false)
+          .setBadRecordRouter(BadRecordRouter.THROWING_ROUTER)
+          .setBadRecordErrorHandler(new ErrorHandler.DefaultErrorHandler<>())
           .build()
           .withProcessingTime()
           .withMonotonicallyIncreasingWatermarkEstimator();
@@ -2305,6 +2352,14 @@ public ReadSourceDescriptors<K, V> withConsumerConfigOverrides(
       return toBuilder().setConsumerConfig(consumerConfig).build();
     }
 
+    public ReadSourceDescriptors<K, V> withBadRecordErrorHandler(
+        ErrorHandler<BadRecord, ?> errorHandler) {
+      return toBuilder()
+          .setBadRecordRouter(BadRecordRouter.RECORDING_ROUTER)
+          .setBadRecordErrorHandler(errorHandler)
+          .build();
+    }
+
     ReadAllFromRow<K, V> forExternalBuild() {
       return new ReadAllFromRow<>(this);
     }
@@ -2395,9 +2450,18 @@ public PCollection<KafkaRecord<K, V>> expand(PCollection<KafkaSourceDescriptor>
       Coder<KafkaRecord<K, V>> recordCoder = KafkaRecordCoder.of(keyCoder, valueCoder);
 
       try {
+        PCollectionTuple pCollectionTuple =
+            input.apply(
+                ParDo.of(ReadFromKafkaDoFn.<K, V>create(this, records))
+                    .withOutputTags(records, TupleTagList.of(BadRecordRouter.BAD_RECORD_TAG)));
+        getBadRecordErrorHandler()
+            .addErrorCollection(
+                pCollectionTuple
+                    .get(BadRecordRouter.BAD_RECORD_TAG)
+                    .setCoder(BadRecord.getCoder(input.getPipeline())));
         PCollection<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> outputWithDescriptor =
-            input
-                .apply(ParDo.of(ReadFromKafkaDoFn.<K, V>create(this)))
+            pCollectionTuple
+                .get(records)
                 .setCoder(
                     KvCoder.of(
                         input
@@ -2538,6 +2602,12 @@ public abstract static class WriteRecords<K, V>
     public abstract @Nullable SerializableFunction<Map<String, Object>, ? extends Consumer<?, ?>>
         getConsumerFactoryFn();
 
+    @Pure
+    public abstract BadRecordRouter getBadRecordRouter();
+
+    @Pure
+    public abstract ErrorHandler<BadRecord, ?> getBadRecordErrorHandler();
+
     abstract Builder<K, V> toBuilder();
 
     @AutoValue.Builder
@@ -2565,6 +2635,11 @@ abstract Builder<K, V> setPublishTimestampFunction(
       abstract Builder<K, V> setConsumerFactoryFn(
           SerializableFunction<Map<String, Object>, ? extends Consumer<?, ?>> fn);
 
+      abstract Builder<K, V> setBadRecordRouter(BadRecordRouter router);
+
+      abstract Builder<K, V> setBadRecordErrorHandler(
+          ErrorHandler<BadRecord, ?> badRecordErrorHandler);
+
       abstract WriteRecords<K, V> build();
     }
 
@@ -2711,6 +2786,14 @@ public WriteRecords<K, V> withConsumerFactoryFn(
       return toBuilder().setConsumerFactoryFn(consumerFactoryFn).build();
     }
 
+    public WriteRecords<K, V> withBadRecordErrorHandler(
+        ErrorHandler<BadRecord, ?> badRecordErrorHandler) {
+      return toBuilder()
+          .setBadRecordRouter(BadRecordRouter.RECORDING_ROUTER)
+          .setBadRecordErrorHandler(badRecordErrorHandler)
+          .build();
+    }
+
     @Override
     public PDone expand(PCollection<ProducerRecord<K, V>> input) {
       checkArgument(
@@ -2722,6 +2805,9 @@ public PDone expand(PCollection<ProducerRecord<K, V>> input) {
 
       if (isEOS()) {
         checkArgument(getTopic() != null, "withTopic() is required when isEOS() is true");
+        checkArgument(
+            getBadRecordErrorHandler() instanceof DefaultErrorHandler,
+            "BadRecordErrorHandling isn't supported with Kafka Exactly Once writing");
         KafkaExactlyOnceSink.ensureEOSSupport();
 
         // TODO: Verify that the group_id does not have existing state stored on Kafka unless
@@ -2732,7 +2818,19 @@ public PDone expand(PCollection<ProducerRecord<K, V>> input) {
 
         input.apply(new KafkaExactlyOnceSink<>(this));
       } else {
-        input.apply(ParDo.of(new KafkaWriter<>(this)));
+        // Even though the errors are the only output from writing to Kafka, we maintain a
+        // PCollectionTuple
+        // with a void tag as the 'primary' output for easy forward compatibility
+        PCollectionTuple pCollectionTuple =
+            input.apply(
+                ParDo.of(new KafkaWriter<>(this))
+                    .withOutputTags(
+                        new TupleTag<Void>(), TupleTagList.of(BadRecordRouter.BAD_RECORD_TAG)));
+        getBadRecordErrorHandler()
+            .addErrorCollection(
+                pCollectionTuple
+                    .get(BadRecordRouter.BAD_RECORD_TAG)
+                    .setCoder(BadRecord.getCoder(input.getPipeline())));
       }
       return PDone.in(input.getPipeline());
     }
@@ -2995,6 +3093,15 @@ public Write<K, V> withProducerConfigUpdates(Map<String, Object> configUpdates)
           getWriteRecordsTransform().withProducerConfigUpdates(configUpdates));
     }
 
+    /**
+     * Configure a {@link BadRecordErrorHandler} for sending records to if they fail to serialize
+     * when being sent to Kafka.
+     */
+    public Write<K, V> withBadRecordErrorHandler(ErrorHandler<BadRecord, ?> badRecordErrorHandler) {
+      return withWriteRecordsTransform(
+          getWriteRecordsTransform().withBadRecordErrorHandler(badRecordErrorHandler));
+    }
+
     @Override
     public PDone expand(PCollection<KV<K, V>> input) {
       final String topic = Preconditions.checkStateNotNull(getTopic(), "withTopic() is required");
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java
index b779de1d9cf1..a2cc9aaeb4d9 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaIOReadImplementationCompatibility.java
@@ -111,6 +111,7 @@ Object getDefaultValue() {
     KEY_DESERIALIZER_PROVIDER,
     VALUE_DESERIALIZER_PROVIDER,
     CHECK_STOP_READING_FN(SDF),
+    BAD_RECORD_ERROR_HANDLER(SDF),
     ;
 
     @Nonnull private final ImmutableSet<KafkaIOReadImplementation> supportedImplementations;
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriter.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriter.java
index c0c9772959f9..4f4663aa8cc8 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriter.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/KafkaWriter.java
@@ -25,6 +25,7 @@
 import org.apache.beam.sdk.metrics.Counter;
 import org.apache.beam.sdk.metrics.SinkMetrics;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter;
 import org.apache.beam.sdk.util.Preconditions;
 import org.apache.kafka.clients.producer.Callback;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -32,6 +33,7 @@
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.errors.SerializationException;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -57,7 +59,7 @@ public void setup() {
   // Suppression since errors are tracked in SendCallback(), and checked in finishBundle()
   @ProcessElement
   @SuppressWarnings("FutureReturnValueIgnored")
-  public void processElement(ProcessContext ctx) throws Exception {
+  public void processElement(ProcessContext ctx, MultiOutputReceiver receiver) throws Exception {
     Producer<K, V> producer = Preconditions.checkStateNotNull(this.producer);
     checkForFailures();
 
@@ -75,19 +77,31 @@ public void processElement(ProcessContext ctx) throws Exception {
       topicName = spec.getTopic();
     }
 
-    @SuppressWarnings({"nullness", "unused"}) // Kafka library not annotated
-    Future<RecordMetadata> ignored =
-        producer.send(
-            new ProducerRecord<>(
-                topicName,
-                record.partition(),
-                timestampMillis,
-                record.key(),
-                record.value(),
-                record.headers()),
-            callback);
-
-    elementsWritten.inc();
+    try {
+      @SuppressWarnings({"nullness", "unused"}) // Kafka library not annotated
+      Future<RecordMetadata> ignored =
+          producer.send(
+              new ProducerRecord<>(
+                  topicName,
+                  record.partition(),
+                  timestampMillis,
+                  record.key(),
+                  record.value(),
+                  record.headers()),
+              callback);
+
+      elementsWritten.inc();
+    } catch (SerializationException e) {
+      // This exception should only occur during the key and value deserialization when
+      // creating the Kafka Record. We can catch the exception here as producer.send serializes
+      // the record before starting the future.
+      badRecordRouter.route(
+          receiver,
+          record,
+          null,
+          e,
+          "Failure serializing Key or Value of Kakfa record writing from Kafka");
+    }
   }
 
   @FinishBundle
@@ -110,6 +124,8 @@ public void teardown() {
   private final WriteRecords<K, V> spec;
   private final Map<String, Object> producerConfig;
 
+  private final BadRecordRouter badRecordRouter;
+
   private transient @Nullable Producer<K, V> producer = null;
   // first exception and number of failures since last invocation of checkForFailures():
   private transient @Nullable Exception sendException = null;
@@ -122,6 +138,8 @@ public void teardown() {
 
     this.producerConfig = new HashMap<>(spec.getProducerConfig());
 
+    this.badRecordRouter = spec.getBadRecordRouter();
+
     if (spec.getKeySerializer() != null) {
       this.producerConfig.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, spec.getKeySerializer());
     }
diff --git a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java
index 1b6e3addce22..924833290f13 100644
--- a/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java
+++ b/sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFn.java
@@ -35,6 +35,7 @@
 import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter;
 import org.apache.beam.sdk.transforms.splittabledofn.GrowableOffsetRangeTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
 import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
@@ -45,6 +46,7 @@
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.util.Preconditions;
 import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Stopwatch;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Supplier;
@@ -60,6 +62,7 @@
 import org.apache.kafka.clients.consumer.ConsumerRecords;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.joda.time.Instant;
@@ -144,29 +147,37 @@
 abstract class ReadFromKafkaDoFn<K, V>
     extends DoFn<KafkaSourceDescriptor, KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> {
 
-  static <K, V> ReadFromKafkaDoFn<K, V> create(ReadSourceDescriptors<K, V> transform) {
+  static <K, V> ReadFromKafkaDoFn<K, V> create(
+      ReadSourceDescriptors<K, V> transform,
+      TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag) {
     if (transform.isBounded()) {
-      return new Bounded<>(transform);
+      return new Bounded<>(transform, recordTag);
     } else {
-      return new Unbounded<>(transform);
+      return new Unbounded<>(transform, recordTag);
     }
   }
 
   @UnboundedPerElement
   private static class Unbounded<K, V> extends ReadFromKafkaDoFn<K, V> {
-    Unbounded(ReadSourceDescriptors<K, V> transform) {
-      super(transform);
+    Unbounded(
+        ReadSourceDescriptors<K, V> transform,
+        TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag) {
+      super(transform, recordTag);
     }
   }
 
   @BoundedPerElement
   private static class Bounded<K, V> extends ReadFromKafkaDoFn<K, V> {
-    Bounded(ReadSourceDescriptors<K, V> transform) {
-      super(transform);
+    Bounded(
+        ReadSourceDescriptors<K, V> transform,
+        TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag) {
+      super(transform, recordTag);
     }
   }
 
-  private ReadFromKafkaDoFn(ReadSourceDescriptors<K, V> transform) {
+  private ReadFromKafkaDoFn(
+      ReadSourceDescriptors<K, V> transform,
+      TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag) {
     this.consumerConfig = transform.getConsumerConfig();
     this.offsetConsumerConfig = transform.getOffsetConsumerConfig();
     this.keyDeserializerProvider =
@@ -178,6 +189,8 @@ private ReadFromKafkaDoFn(ReadSourceDescriptors<K, V> transform) {
     this.createWatermarkEstimatorFn = transform.getCreateWatermarkEstimatorFn();
     this.timestampPolicyFactory = transform.getTimestampPolicyFactory();
     this.checkStopReadingFn = transform.getCheckStopReadingFn();
+    this.badRecordRouter = transform.getBadRecordRouter();
+    this.recordTag = recordTag;
   }
 
   private static final Logger LOG = LoggerFactory.getLogger(ReadFromKafkaDoFn.class);
@@ -193,6 +206,10 @@ private ReadFromKafkaDoFn(ReadSourceDescriptors<K, V> transform) {
       createWatermarkEstimatorFn;
   private final @Nullable TimestampPolicyFactory<K, V> timestampPolicyFactory;
 
+  private final BadRecordRouter badRecordRouter;
+
+  private final TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> recordTag;
+
   // Valid between bundle start and bundle finish.
   private transient @Nullable Deserializer<K> keyDeserializerInstance = null;
   private transient @Nullable Deserializer<V> valueDeserializerInstance = null;
@@ -361,7 +378,8 @@ public ProcessContinuation processElement(
       @Element KafkaSourceDescriptor kafkaSourceDescriptor,
       RestrictionTracker<OffsetRange, Long> tracker,
       WatermarkEstimator<Instant> watermarkEstimator,
-      OutputReceiver<KV<KafkaSourceDescriptor, KafkaRecord<K, V>>> receiver) {
+      MultiOutputReceiver receiver)
+      throws Exception {
     final LoadingCache<TopicPartition, AverageRecordSize> avgRecordSize =
         Preconditions.checkStateNotNull(this.avgRecordSize);
     final Deserializer<K> keyDeserializerInstance =
@@ -431,36 +449,52 @@ public ProcessContinuation processElement(
           if (!tracker.tryClaim(rawRecord.offset())) {
             return ProcessContinuation.stop();
           }
-          KafkaRecord<K, V> kafkaRecord =
-              new KafkaRecord<>(
-                  rawRecord.topic(),
-                  rawRecord.partition(),
-                  rawRecord.offset(),
-                  ConsumerSpEL.getRecordTimestamp(rawRecord),
-                  ConsumerSpEL.getRecordTimestampType(rawRecord),
-                  ConsumerSpEL.hasHeaders() ? rawRecord.headers() : null,
-                  ConsumerSpEL.deserializeKey(keyDeserializerInstance, rawRecord),
-                  ConsumerSpEL.deserializeValue(valueDeserializerInstance, rawRecord));
-          int recordSize =
-              (rawRecord.key() == null ? 0 : rawRecord.key().length)
-                  + (rawRecord.value() == null ? 0 : rawRecord.value().length);
-          avgRecordSize
-              .getUnchecked(kafkaSourceDescriptor.getTopicPartition())
-              .update(recordSize, rawRecord.offset() - expectedOffset);
-          rawSizes.update(recordSize);
-          expectedOffset = rawRecord.offset() + 1;
-          Instant outputTimestamp;
-          // The outputTimestamp and watermark will be computed by timestampPolicy, where the
-          // WatermarkEstimator should be a manual one.
-          if (timestampPolicy != null) {
-            TimestampPolicyContext context =
-                updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
-            outputTimestamp = timestampPolicy.getTimestampForRecord(context, kafkaRecord);
-          } else {
-            Preconditions.checkStateNotNull(this.extractOutputTimestampFn);
-            outputTimestamp = extractOutputTimestampFn.apply(kafkaRecord);
+          try {
+            KafkaRecord<K, V> kafkaRecord =
+                new KafkaRecord<>(
+                    rawRecord.topic(),
+                    rawRecord.partition(),
+                    rawRecord.offset(),
+                    ConsumerSpEL.getRecordTimestamp(rawRecord),
+                    ConsumerSpEL.getRecordTimestampType(rawRecord),
+                    ConsumerSpEL.hasHeaders() ? rawRecord.headers() : null,
+                    ConsumerSpEL.deserializeKey(keyDeserializerInstance, rawRecord),
+                    ConsumerSpEL.deserializeValue(valueDeserializerInstance, rawRecord));
+            int recordSize =
+                (rawRecord.key() == null ? 0 : rawRecord.key().length)
+                    + (rawRecord.value() == null ? 0 : rawRecord.value().length);
+            avgRecordSize
+                .getUnchecked(kafkaSourceDescriptor.getTopicPartition())
+                .update(recordSize, rawRecord.offset() - expectedOffset);
+            rawSizes.update(recordSize);
+            expectedOffset = rawRecord.offset() + 1;
+            Instant outputTimestamp;
+            // The outputTimestamp and watermark will be computed by timestampPolicy, where the
+            // WatermarkEstimator should be a manual one.
+            if (timestampPolicy != null) {
+              TimestampPolicyContext context =
+                  updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
+              outputTimestamp = timestampPolicy.getTimestampForRecord(context, kafkaRecord);
+            } else {
+              Preconditions.checkStateNotNull(this.extractOutputTimestampFn);
+              outputTimestamp = extractOutputTimestampFn.apply(kafkaRecord);
+            }
+            receiver
+                .get(recordTag)
+                .outputWithTimestamp(KV.of(kafkaSourceDescriptor, kafkaRecord), outputTimestamp);
+          } catch (SerializationException e) {
+            // This exception should only occur during the key and value deserialization when
+            // creating the Kafka Record
+            badRecordRouter.route(
+                receiver,
+                rawRecord,
+                null,
+                e,
+                "Failure deserializing Key or Value of Kakfa record reading from Kafka");
+            if (timestampPolicy != null) {
+              updateWatermarkManually(timestampPolicy, watermarkEstimator, tracker);
+            }
           }
-          receiver.outputWithTimestamp(KV.of(kafkaSourceDescriptor, kafkaRecord), outputTimestamp);
         }
       }
     }
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java
index 2ccf7dcc3a93..38bf723a15a9 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOExternalTest.java
@@ -350,13 +350,7 @@ public void testConstructKafkaWrite() throws Exception {
     RunnerApi.PTransform writeComposite =
         result.getComponents().getTransformsOrThrow(transform.getSubtransforms(1));
     RunnerApi.PTransform writeParDo =
-        result
-            .getComponents()
-            .getTransformsOrThrow(
-                result
-                    .getComponents()
-                    .getTransformsOrThrow(writeComposite.getSubtransforms(0))
-                    .getSubtransforms(0));
+        result.getComponents().getTransformsOrThrow(writeComposite.getSubtransforms(0));
 
     RunnerApi.ParDoPayload parDoPayload =
         RunnerApi.ParDoPayload.parseFrom(writeParDo.getSpec().getPayload());
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
index 2c8ace9c66c1..ab6ac52e318d 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOIT.java
@@ -29,6 +29,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
@@ -43,6 +44,8 @@
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.io.common.IOITHelper;
 import org.apache.beam.sdk.io.common.IOTestPipelineOptions;
+import org.apache.beam.sdk.io.kafka.KafkaIOTest.FailingLongSerializer;
+import org.apache.beam.sdk.io.kafka.ReadFromKafkaDoFnTest.FailingDeserializer;
 import org.apache.beam.sdk.io.synthetic.SyntheticBoundedSource;
 import org.apache.beam.sdk.io.synthetic.SyntheticSourceOptions;
 import org.apache.beam.sdk.options.Default;
@@ -72,6 +75,8 @@
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.Values;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandlingTestUtils.ErrorSinkTransform;
 import org.apache.beam.sdk.transforms.windowing.CalendarWindows;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -124,8 +129,6 @@ public class KafkaIOIT {
 
   private static final String RUN_TIME_METRIC_NAME = "run_time";
 
-  private static final String READ_ELEMENT_METRIC_NAME = "kafka_read_element_count";
-
   private static final String NAMESPACE = KafkaIOIT.class.getName();
 
   private static final String TEST_ID = UUID.randomUUID().toString();
@@ -352,6 +355,68 @@ public void processElement(@Element String element, OutputReceiver<String> outpu
     }
   }
 
+  // This test verifies that bad data from Kafka is properly sent to the error handler
+  @Test
+  public void testKafkaIOSDFReadWithErrorHandler() throws IOException {
+    writePipeline
+        .apply(Create.of(KV.of("key", "val")))
+        .apply(
+            "Write to Kafka",
+            KafkaIO.<String, String>write()
+                .withBootstrapServers(options.getKafkaBootstrapServerAddresses())
+                .withKeySerializer(StringSerializer.class)
+                .withValueSerializer(StringSerializer.class)
+                .withTopic(options.getKafkaTopic() + "-failingDeserialization"));
+
+    PipelineResult writeResult = writePipeline.run();
+    PipelineResult.State writeState = writeResult.waitUntilFinish();
+    assertNotEquals(PipelineResult.State.FAILED, writeState);
+
+    BadRecordErrorHandler<PCollection<Long>> eh =
+        sdfReadPipeline.registerBadRecordErrorHandler(new ErrorSinkTransform());
+    sdfReadPipeline.apply(
+        KafkaIO.<String, String>read()
+            .withBootstrapServers(options.getKafkaBootstrapServerAddresses())
+            .withTopic(options.getKafkaTopic() + "-failingDeserialization")
+            .withConsumerConfigUpdates(ImmutableMap.of("auto.offset.reset", "earliest"))
+            .withKeyDeserializer(FailingDeserializer.class)
+            .withValueDeserializer(FailingDeserializer.class)
+            .withBadRecordErrorHandler(eh));
+    eh.close();
+
+    PAssert.thatSingleton(Objects.requireNonNull(eh.getOutput())).isEqualTo(1L);
+
+    PipelineResult readResult = sdfReadPipeline.run();
+    PipelineResult.State readState =
+        readResult.waitUntilFinish(Duration.standardSeconds(options.getReadTimeout()));
+    cancelIfTimeouted(readResult, readState);
+    assertNotEquals(PipelineResult.State.FAILED, readState);
+  }
+
+  @Test
+  public void testKafkaIOWriteWithErrorHandler() throws IOException {
+
+    BadRecordErrorHandler<PCollection<Long>> eh =
+        writePipeline.registerBadRecordErrorHandler(new ErrorSinkTransform());
+    writePipeline
+        .apply("Create single KV", Create.of(KV.of("key", 4L)))
+        .apply(
+            "Write to Kafka",
+            KafkaIO.<String, Long>write()
+                .withBootstrapServers(options.getKafkaBootstrapServerAddresses())
+                .withKeySerializer(StringSerializer.class)
+                .withValueSerializer(FailingLongSerializer.class)
+                .withTopic(options.getKafkaTopic() + "-failingSerialization")
+                .withBadRecordErrorHandler(eh));
+    eh.close();
+
+    PAssert.thatSingleton(Objects.requireNonNull(eh.getOutput())).isEqualTo(1L);
+
+    PipelineResult writeResult = writePipeline.run();
+    PipelineResult.State writeState = writeResult.waitUntilFinish();
+    assertNotEquals(PipelineResult.State.FAILED, writeState);
+  }
+
   // This test roundtrips a single KV<Null,Null> to verify that externalWithMetadata
   // can handle null keys and values correctly.
   @Test
@@ -484,9 +549,7 @@ public void testKafkaWithDynamicPartitions() throws IOException {
   public void testKafkaWithStopReadingFunction() {
     AlwaysStopCheckStopReadingFn checkStopReadingFn = new AlwaysStopCheckStopReadingFn();
 
-    PipelineResult readResult = runWithStopReadingFn(checkStopReadingFn, "stop-reading");
-
-    assertEquals(-1, readElementMetric(readResult, NAMESPACE, READ_ELEMENT_METRIC_NAME));
+    runWithStopReadingFn(checkStopReadingFn, "stop-reading", 0L);
   }
 
   private static class AlwaysStopCheckStopReadingFn implements CheckStopReadingFn {
@@ -500,11 +563,7 @@ public Boolean apply(TopicPartition input) {
   public void testKafkaWithDelayedStopReadingFunction() {
     DelayedCheckStopReadingFn checkStopReadingFn = new DelayedCheckStopReadingFn();
 
-    PipelineResult readResult = runWithStopReadingFn(checkStopReadingFn, "delayed-stop-reading");
-
-    assertEquals(
-        sourceOptions.numRecords,
-        readElementMetric(readResult, NAMESPACE, READ_ELEMENT_METRIC_NAME));
+    runWithStopReadingFn(checkStopReadingFn, "delayed-stop-reading", sourceOptions.numRecords);
   }
 
   public static final Schema KAFKA_TOPIC_SCHEMA =
@@ -644,7 +703,7 @@ private static class DelayedCheckStopReadingFn implements CheckStopReadingFn {
 
     @Override
     public Boolean apply(TopicPartition input) {
-      if (checkCount >= 5) {
+      if (checkCount >= 10) {
         return true;
       }
       checkCount++;
@@ -652,7 +711,8 @@ public Boolean apply(TopicPartition input) {
     }
   }
 
-  private PipelineResult runWithStopReadingFn(CheckStopReadingFn function, String topicSuffix) {
+  private void runWithStopReadingFn(
+      CheckStopReadingFn function, String topicSuffix, Long expectedCount) {
     writePipeline
         .apply("Generate records", Read.from(new SyntheticBoundedSource(sourceOptions)))
         .apply("Measure write time", ParDo.of(new TimeMonitor<>(NAMESPACE, WRITE_TIME_METRIC_NAME)))
@@ -661,21 +721,31 @@ private PipelineResult runWithStopReadingFn(CheckStopReadingFn function, String
             writeToKafka().withTopic(options.getKafkaTopic() + "-" + topicSuffix));
 
     readPipeline.getOptions().as(Options.class).setStreaming(true);
-    readPipeline
-        .apply(
-            "Read from unbounded Kafka",
-            readFromKafka()
-                .withTopic(options.getKafkaTopic() + "-" + topicSuffix)
-                .withCheckStopReadingFn(function))
-        .apply("Measure read time", ParDo.of(new TimeMonitor<>(NAMESPACE, READ_TIME_METRIC_NAME)));
+    PCollection<Long> count =
+        readPipeline
+            .apply(
+                "Read from unbounded Kafka",
+                readFromKafka()
+                    .withTopic(options.getKafkaTopic() + "-" + topicSuffix)
+                    .withCheckStopReadingFn(function))
+            .apply(
+                "Measure read time", ParDo.of(new TimeMonitor<>(NAMESPACE, READ_TIME_METRIC_NAME)))
+            .apply("Window", Window.into(CalendarWindows.years(1)))
+            .apply(
+                "Counting element",
+                Combine.globally(Count.<KafkaRecord<byte[], byte[]>>combineFn()).withoutDefaults());
+
+    if (expectedCount == 0L) {
+      PAssert.that(count).empty();
+    } else {
+      PAssert.thatSingleton(count).isEqualTo(expectedCount);
+    }
 
     PipelineResult writeResult = writePipeline.run();
     writeResult.waitUntilFinish();
 
     PipelineResult readResult = readPipeline.run();
     readResult.waitUntilFinish(Duration.standardSeconds(options.getReadTimeout()));
-
-    return readResult;
   }
 
   @Test
@@ -686,7 +756,7 @@ public void testWatermarkUpdateWithSparseMessages() throws IOException, Interrup
 
     String topicName = "SparseDataTopicPartition-" + UUID.randomUUID();
     Map<Integer, String> records = new HashMap<>();
-    for (int i = 0; i < 5; i++) {
+    for (int i = 1; i <= 5; i++) {
       records.put(i, String.valueOf(i));
     }
 
@@ -725,7 +795,7 @@ public void testWatermarkUpdateWithSparseMessages() throws IOException, Interrup
 
       PipelineResult readResult = sdfReadPipeline.run();
 
-      Thread.sleep(options.getReadTimeout() * 1000);
+      Thread.sleep(options.getReadTimeout() * 1000 * 2);
 
       for (String value : records.values()) {
         kafkaIOITExpectedLogs.verifyError(value);
@@ -753,11 +823,6 @@ public void processElement(
     }
   }
 
-  private long readElementMetric(PipelineResult result, String namespace, String name) {
-    MetricsReader metricsReader = new MetricsReader(result, namespace);
-    return metricsReader.getCounterMetric(name);
-  }
-
   private Set<NamedTestResult> readMetrics(PipelineResult writeResult, PipelineResult readResult) {
     BiFunction<MetricsReader, String, NamedTestResult> supplier =
         (reader, metricName) -> {
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
index aeb5818e9134..9b15b86051f5 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/KafkaIOTest.java
@@ -51,6 +51,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -99,6 +100,8 @@
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.Values;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.BadRecordErrorHandler;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandlingTestUtils.ErrorSinkTransform;
 import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
 import org.apache.beam.sdk.transforms.windowing.FixedWindows;
 import org.apache.beam.sdk.transforms.windowing.Window;
@@ -121,9 +124,12 @@
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerConfig;
 import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.internals.DefaultPartitioner;
+import org.apache.kafka.common.Cluster;
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.header.Header;
 import org.apache.kafka.common.header.Headers;
 import org.apache.kafka.common.header.internals.RecordHeader;
@@ -1379,7 +1385,7 @@ public void testSink() throws Exception {
 
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1404,13 +1410,69 @@ public void testSink() throws Exception {
     }
   }
 
+  public static class FailingLongSerializer implements Serializer<Long> {
+    // enables instantiation by registrys
+    public FailingLongSerializer() {}
+
+    @Override
+    public byte[] serialize(String topic, Long data) {
+      throw new SerializationException("ExpectedSerializationException");
+    }
+
+    @Override
+    public void configure(Map<String, ?> configs, boolean isKey) {
+      // intentionally left blank for compatibility with older kafka versions
+    }
+  }
+
+  @Test
+  public void testSinkWithSerializationErrors() throws Exception {
+    // Attempt to write 10 elements to Kafka, but they will all fail to serialize, and be sent to
+    // the DLQ
+
+    int numElements = 10;
+
+    try (MockProducerWrapper producerWrapper =
+        new MockProducerWrapper(new FailingLongSerializer())) {
+
+      ProducerSendCompletionThread completionThread =
+          new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
+
+      String topic = "test";
+
+      BadRecordErrorHandler<PCollection<Long>> eh =
+          p.registerBadRecordErrorHandler(new ErrorSinkTransform());
+
+      p.apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata())
+          .apply(
+              KafkaIO.<Integer, Long>write()
+                  .withBootstrapServers("none")
+                  .withTopic(topic)
+                  .withKeySerializer(IntegerSerializer.class)
+                  .withValueSerializer(FailingLongSerializer.class)
+                  .withInputTimestamp()
+                  .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey))
+                  .withBadRecordErrorHandler(eh));
+
+      eh.close();
+
+      PAssert.thatSingleton(Objects.requireNonNull(eh.getOutput())).isEqualTo(10L);
+
+      p.run();
+
+      completionThread.shutdown();
+
+      verifyProducerRecords(producerWrapper.mockProducer, topic, 0, false, true);
+    }
+  }
+
   @Test
   public void testValuesSink() throws Exception {
     // similar to testSink(), but use values()' interface.
 
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1442,7 +1504,7 @@ public void testRecordsSink() throws Exception {
 
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1474,7 +1536,7 @@ public void testSinkToMultipleTopics() throws Exception {
     // Set different output topic names
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1519,7 +1581,7 @@ public void testKafkaWriteHeaders() throws Exception {
     // Set different output topic names
     int numElements = 1;
     SimpleEntry<String, String> header = new SimpleEntry<>("header_key", "header_value");
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1562,7 +1624,7 @@ public void testKafkaWriteHeaders() throws Exception {
   public void testSinkProducerRecordsWithCustomTS() throws Exception {
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1601,7 +1663,7 @@ public void testSinkProducerRecordsWithCustomTS() throws Exception {
   public void testSinkProducerRecordsWithCustomPartition() throws Exception {
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1725,7 +1787,7 @@ public void testExactlyOnceSink() {
 
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -1803,7 +1865,7 @@ public void testSinkWithSendErrors() throws Throwable {
 
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThreadWithErrors =
           new ProducerSendCompletionThread(producerWrapper.mockProducer, 10, 100).start();
@@ -1993,7 +2055,7 @@ public void testSourceWithPatternDisplayData() {
 
   @Test
   public void testSinkDisplayData() {
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
       KafkaIO.Write<Integer, Long> write =
           KafkaIO.<Integer, Long>write()
               .withBootstrapServers("myServerA:9092,myServerB:9092")
@@ -2017,7 +2079,7 @@ public void testSinkMetrics() throws Exception {
 
     int numElements = 1000;
 
-    try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {
+    try (MockProducerWrapper producerWrapper = new MockProducerWrapper(new LongSerializer())) {
 
       ProducerSendCompletionThread completionThread =
           new ProducerSendCompletionThread(producerWrapper.mockProducer).start();
@@ -2109,14 +2171,22 @@ private static class MockProducerWrapper implements AutoCloseable {
       }
     }
 
-    MockProducerWrapper() {
+    MockProducerWrapper(Serializer<Long> valueSerializer) {
       producerKey = String.valueOf(ThreadLocalRandom.current().nextLong());
       mockProducer =
           new MockProducer<Integer, Long>(
+              Cluster.empty()
+                  .withPartitions(
+                      ImmutableMap.of(
+                          new TopicPartition("test", 0),
+                          new PartitionInfo("test", 0, null, null, null),
+                          new TopicPartition("test", 1),
+                          new PartitionInfo("test", 1, null, null, null))),
               false, // disable synchronous completion of send. see ProducerSendCompletionThread
               // below.
+              new DefaultPartitioner(),
               new IntegerSerializer(),
-              new LongSerializer()) {
+              valueSerializer) {
 
             // override flush() so that it does not complete all the waiting sends, giving a chance
             // to
diff --git a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
index 554c6d2fcaf1..845d974af0b4 100644
--- a/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
+++ b/sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/ReadFromKafkaDoFnTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.io.kafka;
 
+import static org.apache.beam.sdk.transforms.errorhandling.BadRecordRouter.BAD_RECORD_TAG;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -41,15 +42,22 @@
 import org.apache.beam.sdk.runners.TransformHierarchy.Node;
 import org.apache.beam.sdk.testing.TestPipeline;
 import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn.MultiOutputReceiver;
 import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
 import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.errorhandling.BadRecord;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler.DefaultErrorHandler;
 import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollection.IsBounded;
 import org.apache.beam.sdk.values.PValue;
+import org.apache.beam.sdk.values.Row;
+import org.apache.beam.sdk.values.TupleTag;
 import org.apache.beam.sdk.values.TypeDescriptor;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Charsets;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
@@ -64,7 +72,9 @@
 import org.apache.kafka.common.KafkaException;
 import org.apache.kafka.common.PartitionInfo;
 import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.errors.SerializationException;
 import org.apache.kafka.common.header.internals.RecordHeaders;
+import org.apache.kafka.common.serialization.Deserializer;
 import org.apache.kafka.common.serialization.StringDeserializer;
 import org.checkerframework.checker.initialization.qual.Initialized;
 import org.checkerframework.checker.nullness.qual.NonNull;
@@ -80,19 +90,22 @@ public class ReadFromKafkaDoFnTest {
 
   private final TopicPartition topicPartition = new TopicPartition("topic", 0);
 
+  private static final TupleTag<KV<KafkaSourceDescriptor, KafkaRecord<String, String>>> RECORDS =
+      new TupleTag<>();
+
   @Rule public ExpectedException thrown = ExpectedException.none();
 
   private final SimpleMockKafkaConsumer consumer =
       new SimpleMockKafkaConsumer(OffsetResetStrategy.NONE, topicPartition);
 
   private final ReadFromKafkaDoFn<String, String> dofnInstance =
-      ReadFromKafkaDoFn.create(makeReadSourceDescriptor(consumer));
+      ReadFromKafkaDoFn.create(makeReadSourceDescriptor(consumer), RECORDS);
 
   private final ExceptionMockKafkaConsumer exceptionConsumer =
       new ExceptionMockKafkaConsumer(OffsetResetStrategy.NONE, topicPartition);
 
   private final ReadFromKafkaDoFn<String, String> exceptionDofnInstance =
-      ReadFromKafkaDoFn.create(makeReadSourceDescriptor(exceptionConsumer));
+      ReadFromKafkaDoFn.create(makeReadSourceDescriptor(exceptionConsumer), RECORDS);
 
   private ReadSourceDescriptors<String, String> makeReadSourceDescriptor(
       Consumer<byte[], byte[]> kafkaMockConsumer) {
@@ -109,6 +122,31 @@ public Consumer<byte[], byte[]> apply(Map<String, Object> input) {
         .withBootstrapServers("bootstrap_server");
   }
 
+  private ReadSourceDescriptors<String, String> makeFailingReadSourceDescriptor(
+      Consumer<byte[], byte[]> kafkaMockConsumer) {
+    return ReadSourceDescriptors.<String, String>read()
+        .withKeyDeserializer(FailingDeserializer.class)
+        .withValueDeserializer(FailingDeserializer.class)
+        .withConsumerFactoryFn(
+            new SerializableFunction<Map<String, Object>, Consumer<byte[], byte[]>>() {
+              @Override
+              public Consumer<byte[], byte[]> apply(Map<String, Object> input) {
+                return kafkaMockConsumer;
+              }
+            })
+        .withBootstrapServers("bootstrap_server");
+  }
+
+  public static class FailingDeserializer implements Deserializer<String> {
+
+    public FailingDeserializer() {}
+
+    @Override
+    public String deserialize(String topic, byte[] data) {
+      throw new SerializationException("Intentional serialization exception");
+    }
+  }
+
   private static class ExceptionMockKafkaConsumer extends MockConsumer<byte[], byte[]> {
 
     private final TopicPartition topicPartition;
@@ -254,23 +292,66 @@ public synchronized long position(TopicPartition partition) {
     }
   }
 
-  private static class MockOutputReceiver
-      implements OutputReceiver<KV<KafkaSourceDescriptor, KafkaRecord<String, String>>> {
+  private static class MockMultiOutputReceiver implements MultiOutputReceiver {
+
+    MockOutputReceiver<KV<KafkaSourceDescriptor, KafkaRecord<String, String>>> mockOutputReceiver =
+        new MockOutputReceiver<>();
+
+    MockOutputReceiver<BadRecord> badOutputReceiver = new MockOutputReceiver<>();
+
+    @Override
+    public @UnknownKeyFor @NonNull @Initialized <T> OutputReceiver<T> get(
+        @UnknownKeyFor @NonNull @Initialized TupleTag<T> tag) {
+      if (RECORDS.equals(tag)) {
+        return (OutputReceiver<T>) mockOutputReceiver;
+      } else if (BAD_RECORD_TAG.equals(tag)) {
+        return (OutputReceiver<T>) badOutputReceiver;
+      } else {
+        throw new RuntimeException("Invalid Tag");
+      }
+    }
+
+    public List<KV<KafkaSourceDescriptor, KafkaRecord<String, String>>> getGoodRecords() {
+      return mockOutputReceiver.getOutputs();
+    }
+
+    public List<BadRecord> getBadRecords() {
+      return badOutputReceiver.getOutputs();
+    }
+
+    @Override
+    public @UnknownKeyFor @NonNull @Initialized <T>
+        OutputReceiver<@UnknownKeyFor @NonNull @Initialized Row> getRowReceiver(
+            @UnknownKeyFor @NonNull @Initialized TupleTag<T> tag) {
+      return null;
+    }
+  }
+
+  private static class MockOutputReceiver<T> implements OutputReceiver<T> {
 
-    private final List<KV<KafkaSourceDescriptor, KafkaRecord<String, String>>> records =
-        new ArrayList<>();
+    private final List<T> records = new ArrayList<>();
 
     @Override
-    public void output(KV<KafkaSourceDescriptor, KafkaRecord<String, String>> output) {}
+    public void output(T output) {
+      records.add(output);
+    }
 
     @Override
     public void outputWithTimestamp(
-        KV<KafkaSourceDescriptor, KafkaRecord<String, String>> output,
-        @UnknownKeyFor @NonNull @Initialized Instant timestamp) {
+        T output, @UnknownKeyFor @NonNull @Initialized Instant timestamp) {
       records.add(output);
     }
 
-    public List<KV<KafkaSourceDescriptor, KafkaRecord<String, String>>> getOutputs() {
+    @Override
+    public void outputWindowedValue(
+        T output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      throw new UnsupportedOperationException("Not expecting outputWindowedValue");
+    }
+
+    public List<T> getOutputs() {
       return this.records;
     }
   }
@@ -386,7 +467,7 @@ public void testInitialRestrictionWithException() throws Exception {
 
   @Test
   public void testProcessElement() throws Exception {
-    MockOutputReceiver receiver = new MockOutputReceiver();
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
     consumer.setNumOfRecordsPerPoll(3L);
     long startOffset = 5L;
     OffsetRangeTracker tracker =
@@ -396,7 +477,8 @@ public void testProcessElement() throws Exception {
     ProcessContinuation result = dofnInstance.processElement(descriptor, tracker, null, receiver);
     assertEquals(ProcessContinuation.stop(), result);
     assertEquals(
-        createExpectedRecords(descriptor, startOffset, 3, "key", "value"), receiver.getOutputs());
+        createExpectedRecords(descriptor, startOffset, 3, "key", "value"),
+        receiver.getGoodRecords());
   }
 
   @Test
@@ -406,7 +488,7 @@ public void testRawSizeMetric() throws Exception {
     MetricsContainerImpl container = new MetricsContainerImpl("any");
     MetricsEnvironment.setCurrentContainer(container);
 
-    MockOutputReceiver receiver = new MockOutputReceiver();
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
     consumer.setNumOfRecordsPerPoll(numElements);
     OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0, numElements));
     KafkaSourceDescriptor descriptor =
@@ -427,7 +509,7 @@ public void testRawSizeMetric() throws Exception {
 
   @Test
   public void testProcessElementWithEmptyPoll() throws Exception {
-    MockOutputReceiver receiver = new MockOutputReceiver();
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
     consumer.setNumOfRecordsPerPoll(-1);
     OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0L, Long.MAX_VALUE));
     ProcessContinuation result =
@@ -437,12 +519,12 @@ public void testProcessElementWithEmptyPoll() throws Exception {
             null,
             receiver);
     assertEquals(ProcessContinuation.resume(), result);
-    assertTrue(receiver.getOutputs().isEmpty());
+    assertTrue(receiver.getGoodRecords().isEmpty());
   }
 
   @Test
   public void testProcessElementWhenTopicPartitionIsRemoved() throws Exception {
-    MockOutputReceiver receiver = new MockOutputReceiver();
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
     consumer.setRemoved();
     consumer.setNumOfRecordsPerPoll(10);
     OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0L, Long.MAX_VALUE));
@@ -457,7 +539,7 @@ public void testProcessElementWhenTopicPartitionIsRemoved() throws Exception {
 
   @Test
   public void testProcessElementWhenTopicPartitionIsStopped() throws Exception {
-    MockOutputReceiver receiver = new MockOutputReceiver();
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
     ReadFromKafkaDoFn<String, String> instance =
         ReadFromKafkaDoFn.create(
             makeReadSourceDescriptor(consumer)
@@ -470,7 +552,8 @@ public Boolean apply(TopicPartition input) {
                         return true;
                       }
                     })
-                .build());
+                .build(),
+            RECORDS);
     instance.setup();
     consumer.setNumOfRecordsPerPoll(10);
     OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0L, Long.MAX_VALUE));
@@ -489,7 +572,7 @@ public void testProcessElementWithException() throws Exception {
     thrown.expect(KafkaException.class);
     thrown.expectMessage("SeekException");
 
-    MockOutputReceiver receiver = new MockOutputReceiver();
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
     OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0L, Long.MAX_VALUE));
 
     exceptionDofnInstance.processElement(
@@ -499,6 +582,61 @@ public void testProcessElementWithException() throws Exception {
         receiver);
   }
 
+  @Test
+  public void testProcessElementWithDeserializationExceptionDefaultRecordHandler()
+      throws Exception {
+    thrown.expect(SerializationException.class);
+    thrown.expectMessage("Intentional serialization exception");
+
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
+    OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0L, Long.MAX_VALUE));
+
+    consumer.setNumOfRecordsPerPoll(1);
+
+    ReadFromKafkaDoFn<String, String> dofnInstance =
+        ReadFromKafkaDoFn.create(makeFailingReadSourceDescriptor(consumer), RECORDS);
+
+    dofnInstance.setup();
+
+    dofnInstance.processElement(
+        KafkaSourceDescriptor.of(topicPartition, null, null, null, null, null),
+        tracker,
+        null,
+        receiver);
+
+    Assert.assertEquals("OutputRecordSize", 0, receiver.getGoodRecords().size());
+    Assert.assertEquals("OutputErrorSize", 0, receiver.getBadRecords().size());
+  }
+
+  @Test
+  public void testProcessElementWithDeserializationExceptionRecordingRecordHandler()
+      throws Exception {
+    MockMultiOutputReceiver receiver = new MockMultiOutputReceiver();
+    OffsetRangeTracker tracker = new OffsetRangeTracker(new OffsetRange(0L, 1L));
+
+    consumer.setNumOfRecordsPerPoll(1);
+
+    // Because we never actually execute the pipeline, no data will actually make it to the error
+    // handler. This will just configure the ReadSourceDesriptors to route the errors to the output
+    // PCollection instead of rethrowing.
+    ReadSourceDescriptors<String, String> descriptors =
+        makeFailingReadSourceDescriptor(consumer)
+            .withBadRecordErrorHandler(new DefaultErrorHandler<>());
+
+    ReadFromKafkaDoFn<String, String> dofnInstance = ReadFromKafkaDoFn.create(descriptors, RECORDS);
+
+    dofnInstance.setup();
+
+    dofnInstance.processElement(
+        KafkaSourceDescriptor.of(topicPartition, null, null, null, null, null),
+        tracker,
+        null,
+        receiver);
+
+    Assert.assertEquals("OutputRecordSize", 0, receiver.getGoodRecords().size());
+    Assert.assertEquals("OutputErrorSize", 1, receiver.getBadRecords().size());
+  }
+
   private static final TypeDescriptor<KafkaSourceDescriptor>
       KAFKA_SOURCE_DESCRIPTOR_TYPE_DESCRIPTOR = new TypeDescriptor<KafkaSourceDescriptor>() {};
 
@@ -522,7 +660,8 @@ private BoundednessVisitor testBoundedness(
         .apply(
             ParDo.of(
                 ReadFromKafkaDoFn.<String, String>create(
-                    readSourceDescriptorsDecorator.apply(makeReadSourceDescriptor(consumer)))))
+                    readSourceDescriptorsDecorator.apply(makeReadSourceDescriptor(consumer)),
+                    RECORDS)))
         .setCoder(
             KvCoder.of(
                 SerializableCoder.of(KafkaSourceDescriptor.class),
diff --git a/sdks/java/io/kafka/upgrade/build.gradle b/sdks/java/io/kafka/upgrade/build.gradle
index 78776e6e8264..ca2823740dc6 100644
--- a/sdks/java/io/kafka/upgrade/build.gradle
+++ b/sdks/java/io/kafka/upgrade/build.gradle
@@ -16,8 +16,6 @@
  * limitations under the License.
  */
 
-import java.util.stream.Collectors
-
 plugins { id 'org.apache.beam.module' }
 applyJavaNature(
         automaticModuleName: 'org.apache.beam.sdk.io.kafka.upgrade',
diff --git a/sdks/java/io/kafka/upgrade/src/main/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslation.java b/sdks/java/io/kafka/upgrade/src/main/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslation.java
index 1757da6a6311..eedd2282b1f7 100644
--- a/sdks/java/io/kafka/upgrade/src/main/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslation.java
+++ b/sdks/java/io/kafka/upgrade/src/main/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslation.java
@@ -17,12 +17,12 @@
  */
 package org.apache.beam.sdk.io.kafka.upgrade;
 
+import static org.apache.beam.runners.core.construction.TransformUpgrader.fromByteArray;
+import static org.apache.beam.runners.core.construction.TransformUpgrader.toByteArray;
+
 import com.google.auto.service.AutoService;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import java.io.InvalidClassException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -49,6 +49,7 @@
 import org.apache.beam.sdk.schemas.logicaltypes.NanosInstant;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.transforms.errorhandling.ErrorHandler;
 import org.apache.beam.sdk.values.Row;
 import org.apache.beam.vendor.grpc.v1p54p0.com.google.protobuf.ByteString;
 import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
@@ -73,25 +74,6 @@ public class KafkaIOTranslation {
   public static final String KAFKA_WRITE_TRANSFORM_URN_V2 =
       "beam:transform:org.apache.beam:kafka_write:v2";
 
-  private static byte[] toByteArray(Object object) {
-    try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
-        ObjectOutputStream out = new ObjectOutputStream(bos)) {
-      out.writeObject(object);
-      return bos.toByteArray();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  private static Object fromByteArray(byte[] bytes) {
-    try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
-        ObjectInputStream in = new ObjectInputStream(bis)) {
-      return in.readObject();
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   static class KafkaIOReadWithMetadataTranslator implements TransformPayloadTranslator<Read<?, ?>> {
 
     static Schema topicPartitionSchema =
@@ -223,162 +205,183 @@ public Row toConfigRow(Read<?, ?> transform) {
       if (transform.getCheckStopReadingFn() != null) {
         fieldValues.put("check_stop_reading_fn", toByteArray(transform.getCheckStopReadingFn()));
       }
+      if (transform.getBadRecordErrorHandler() != null) {
+        throw new RuntimeException(
+            "Upgrading KafkaIO read transforms that have `withBadRecordErrorHandler` property set"
+                + "is not supported yet.");
+      }
 
       return Row.withSchema(schema).withFieldValues(fieldValues).build();
     }
 
     @Override
     public Read<?, ?> fromConfigRow(Row configRow) {
-      Read<?, ?> transform = KafkaIO.read();
-
-      Map<String, byte[]> consumerConfig = configRow.getMap("consumer_config");
-      if (consumerConfig != null) {
-        Map<String, Object> updatedConsumerConfig = new HashMap<>();
-        consumerConfig.forEach(
-            (key, dataBytes) -> {
-              // Adding all allowed properties.
-              if (!KafkaIOUtils.DISALLOWED_CONSUMER_PROPERTIES.containsKey(key)) {
-                if (consumerConfig.get(key) == null) {
-                  throw new IllegalArgumentException(
-                      "Encoded value of the consumer config property " + key + " was null");
+      try {
+        Read<?, ?> transform = KafkaIO.read();
+
+        Map<String, byte[]> consumerConfig = configRow.getMap("consumer_config");
+        if (consumerConfig != null) {
+          Map<String, Object> updatedConsumerConfig = new HashMap<>();
+          consumerConfig.forEach(
+              (key, dataBytes) -> {
+                // Adding all allowed properties.
+                if (!KafkaIOUtils.DISALLOWED_CONSUMER_PROPERTIES.containsKey(key)) {
+                  if (consumerConfig.get(key) == null) {
+                    throw new IllegalArgumentException(
+                        "Encoded value of the consumer config property " + key + " was null");
+                  }
+                  try {
+                    updatedConsumerConfig.put(key, fromByteArray(consumerConfig.get(key)));
+                  } catch (InvalidClassException e) {
+                    throw new RuntimeException(e);
+                  }
                 }
-                updatedConsumerConfig.put(key, fromByteArray(consumerConfig.get(key)));
-              }
-            });
-        transform = transform.withConsumerConfigUpdates(updatedConsumerConfig);
-      }
-      Collection<String> topics = configRow.getArray("topics");
-      if (topics != null) {
-        transform = transform.withTopics(new ArrayList<>(topics));
-      }
-      Collection<Row> topicPartitionRows = configRow.getArray("topic_partitions");
-      if (topicPartitionRows != null) {
-        Collection<TopicPartition> topicPartitions =
-            topicPartitionRows.stream()
-                .map(
-                    row -> {
-                      String topic = row.getString("topic");
-                      if (topic == null) {
-                        throw new IllegalArgumentException("Expected the topic to be not null");
-                      }
-                      Integer partition = row.getInt32("partition");
-                      if (partition == null) {
-                        throw new IllegalArgumentException("Expected the partition to be not null");
-                      }
-                      return new TopicPartition(topic, partition);
-                    })
-                .collect(Collectors.toList());
-        transform = transform.withTopicPartitions(Lists.newArrayList(topicPartitions));
-      }
-      String topicPattern = configRow.getString("topic_pattern");
-      if (topicPattern != null) {
-        transform = transform.withTopicPattern(topicPattern);
-      }
+              });
+          transform = transform.withConsumerConfigUpdates(updatedConsumerConfig);
+        }
+        Collection<String> topics = configRow.getArray("topics");
+        if (topics != null) {
+          transform = transform.withTopics(new ArrayList<>(topics));
+        }
+        Collection<Row> topicPartitionRows = configRow.getArray("topic_partitions");
+        if (topicPartitionRows != null) {
+          Collection<TopicPartition> topicPartitions =
+              topicPartitionRows.stream()
+                  .map(
+                      row -> {
+                        String topic = row.getString("topic");
+                        if (topic == null) {
+                          throw new IllegalArgumentException("Expected the topic to be not null");
+                        }
+                        Integer partition = row.getInt32("partition");
+                        if (partition == null) {
+                          throw new IllegalArgumentException(
+                              "Expected the partition to be not null");
+                        }
+                        return new TopicPartition(topic, partition);
+                      })
+                  .collect(Collectors.toList());
+          transform = transform.withTopicPartitions(Lists.newArrayList(topicPartitions));
+        }
+        String topicPattern = configRow.getString("topic_pattern");
+        if (topicPattern != null) {
+          transform = transform.withTopicPattern(topicPattern);
+        }
 
-      byte[] keyDeserializerProvider = configRow.getBytes("key_deserializer_provider");
-      if (keyDeserializerProvider != null) {
+        byte[] keyDeserializerProvider = configRow.getBytes("key_deserializer_provider");
+        if (keyDeserializerProvider != null) {
+
+          byte[] keyCoder = configRow.getBytes("key_coder");
+          if (keyCoder != null) {
+            transform =
+                transform.withKeyDeserializerProviderAndCoder(
+                    (DeserializerProvider) fromByteArray(keyDeserializerProvider),
+                    (org.apache.beam.sdk.coders.Coder) fromByteArray(keyCoder));
+          } else {
+            transform =
+                transform.withKeyDeserializer(
+                    (DeserializerProvider) fromByteArray(keyDeserializerProvider));
+          }
+        }
 
-        byte[] keyCoder = configRow.getBytes("key_coder");
-        if (keyCoder != null) {
-          transform =
-              transform.withKeyDeserializerProviderAndCoder(
-                  (DeserializerProvider) fromByteArray(keyDeserializerProvider),
-                  (org.apache.beam.sdk.coders.Coder) fromByteArray(keyCoder));
-        } else {
-          transform =
-              transform.withKeyDeserializer(
-                  (DeserializerProvider) fromByteArray(keyDeserializerProvider));
+        byte[] valueDeserializerProvider = configRow.getBytes("value_deserializer_provider");
+        if (valueDeserializerProvider != null) {
+          byte[] valueCoder = configRow.getBytes("value_coder");
+          if (valueCoder != null) {
+            transform =
+                transform.withValueDeserializerProviderAndCoder(
+                    (DeserializerProvider) fromByteArray(valueDeserializerProvider),
+                    (org.apache.beam.sdk.coders.Coder) fromByteArray(valueCoder));
+          } else {
+            transform =
+                transform.withValueDeserializer(
+                    (DeserializerProvider) fromByteArray(valueDeserializerProvider));
+          }
         }
-      }
 
-      byte[] valueDeserializerProvider = configRow.getBytes("value_deserializer_provider");
-      if (valueDeserializerProvider != null) {
-        byte[] valueCoder = configRow.getBytes("value_coder");
-        if (valueCoder != null) {
+        byte[] consumerFactoryFn = configRow.getBytes("consumer_factory_fn");
+        if (consumerFactoryFn != null) {
           transform =
-              transform.withValueDeserializerProviderAndCoder(
-                  (DeserializerProvider) fromByteArray(valueDeserializerProvider),
-                  (org.apache.beam.sdk.coders.Coder) fromByteArray(valueCoder));
-        } else {
+              transform.withConsumerFactoryFn(
+                  (SerializableFunction<Map<String, Object>, Consumer<byte[], byte[]>>)
+                      fromByteArray(consumerFactoryFn));
+        }
+        byte[] watermarkFn = configRow.getBytes("watermark_fn");
+        if (watermarkFn != null) {
+          transform = transform.withWatermarkFn2((SerializableFunction) fromByteArray(watermarkFn));
+        }
+        Long maxNumRecords = configRow.getInt64("max_num_records");
+        if (maxNumRecords != null) {
+          transform = transform.withMaxNumRecords(maxNumRecords);
+        }
+        Duration maxReadTime = configRow.getValue("max_read_time");
+        if (maxReadTime != null) {
           transform =
-              transform.withValueDeserializer(
-                  (DeserializerProvider) fromByteArray(valueDeserializerProvider));
+              transform.withMaxReadTime(org.joda.time.Duration.millis(maxReadTime.toMillis()));
+        }
+        Instant startReadTime = configRow.getValue("start_read_time");
+        if (startReadTime != null) {
+          transform = transform.withStartReadTime(startReadTime);
+        }
+        Instant stopReadTime = configRow.getValue("stop_read_time");
+        if (stopReadTime != null) {
+          transform = transform.withStopReadTime(stopReadTime);
+        }
+        Boolean isCommitOffsetFinalizeEnabled =
+            configRow.getBoolean("is_commit_offset_finalize_enabled");
+        if (isCommitOffsetFinalizeEnabled != null && isCommitOffsetFinalizeEnabled) {
+          transform = transform.commitOffsetsInFinalize();
+        }
+        Boolean isDynamicRead = configRow.getBoolean("is_dynamic_read");
+        if (isDynamicRead != null && isDynamicRead) {
+          Duration watchTopicPartitionDuration =
+              configRow.getValue("watch_topic_partition_duration");
+          if (watchTopicPartitionDuration == null) {
+            throw new IllegalArgumentException(
+                "Expected watchTopicPartitionDuration to be available when isDynamicRead is set to true");
+          }
+          transform =
+              transform.withDynamicRead(
+                  org.joda.time.Duration.millis(watchTopicPartitionDuration.toMillis()));
         }
-      }
 
-      byte[] consumerFactoryFn = configRow.getBytes("consumer_factory_fn");
-      if (consumerFactoryFn != null) {
-        transform =
-            transform.withConsumerFactoryFn(
-                (SerializableFunction<Map<String, Object>, Consumer<byte[], byte[]>>)
-                    fromByteArray(consumerFactoryFn));
-      }
-      byte[] watermarkFn = configRow.getBytes("watermark_fn");
-      if (watermarkFn != null) {
-        transform = transform.withWatermarkFn2((SerializableFunction) fromByteArray(watermarkFn));
-      }
-      Long maxNumRecords = configRow.getInt64("max_num_records");
-      if (maxNumRecords != null) {
-        transform = transform.withMaxNumRecords(maxNumRecords);
-      }
-      Duration maxReadTime = configRow.getValue("max_read_time");
-      if (maxReadTime != null) {
-        transform =
-            transform.withMaxReadTime(org.joda.time.Duration.millis(maxReadTime.toMillis()));
-      }
-      Instant startReadTime = configRow.getValue("start_read_time");
-      if (startReadTime != null) {
-        transform = transform.withStartReadTime(startReadTime);
-      }
-      Instant stopReadTime = configRow.getValue("stop_read_time");
-      if (stopReadTime != null) {
-        transform = transform.withStopReadTime(stopReadTime);
-      }
-      Boolean isCommitOffsetFinalizeEnabled =
-          configRow.getBoolean("is_commit_offset_finalize_enabled");
-      if (isCommitOffsetFinalizeEnabled != null && isCommitOffsetFinalizeEnabled) {
-        transform = transform.commitOffsetsInFinalize();
-      }
-      Boolean isDynamicRead = configRow.getBoolean("is_dynamic_read");
-      if (isDynamicRead != null && isDynamicRead) {
-        Duration watchTopicPartitionDuration = configRow.getValue("watch_topic_partition_duration");
-        if (watchTopicPartitionDuration == null) {
-          throw new IllegalArgumentException(
-              "Expected watchTopicPartitionDuration to be available when isDynamicRead is set to true");
+        byte[] timestampPolicyFactory = configRow.getBytes("timestamp_policy_factory");
+        if (timestampPolicyFactory != null) {
+          transform =
+              transform.withTimestampPolicyFactory(
+                  (TimestampPolicyFactory) fromByteArray(timestampPolicyFactory));
+        }
+        Map<String, byte[]> offsetConsumerConfig = configRow.getMap("offset_consumer_config");
+        if (offsetConsumerConfig != null) {
+          Map<String, Object> updatedOffsetConsumerConfig = new HashMap<>();
+          offsetConsumerConfig.forEach(
+              (key, dataBytes) -> {
+                if (offsetConsumerConfig.get(key) == null) {
+                  throw new IllegalArgumentException(
+                      "Encoded value for the offset consumer config key " + key + " was null.");
+                }
+                try {
+                  updatedOffsetConsumerConfig.put(
+                      key, fromByteArray(offsetConsumerConfig.get(key)));
+                } catch (InvalidClassException e) {
+                  throw new RuntimeException(e);
+                }
+              });
+          transform = transform.withOffsetConsumerConfigOverrides(updatedOffsetConsumerConfig);
         }
-        transform =
-            transform.withDynamicRead(
-                org.joda.time.Duration.millis(watchTopicPartitionDuration.toMillis()));
-      }
 
-      byte[] timestampPolicyFactory = configRow.getBytes("timestamp_policy_factory");
-      if (timestampPolicyFactory != null) {
-        transform =
-            transform.withTimestampPolicyFactory(
-                (TimestampPolicyFactory) fromByteArray(timestampPolicyFactory));
-      }
-      Map<String, byte[]> offsetConsumerConfig = configRow.getMap("offset_consumer_config");
-      if (offsetConsumerConfig != null) {
-        Map<String, Object> updatedOffsetConsumerConfig = new HashMap<>();
-        offsetConsumerConfig.forEach(
-            (key, dataBytes) -> {
-              if (offsetConsumerConfig.get(key) == null) {
-                throw new IllegalArgumentException(
-                    "Encoded value for the offset consumer config key " + key + " was null.");
-              }
-              updatedOffsetConsumerConfig.put(key, fromByteArray(offsetConsumerConfig.get(key)));
-            });
-        transform = transform.withOffsetConsumerConfigOverrides(updatedOffsetConsumerConfig);
-      }
+        byte[] checkStopReadinfFn = configRow.getBytes("check_stop_reading_fn");
+        if (checkStopReadinfFn != null) {
+          transform =
+              transform.withCheckStopReadingFn(
+                  (SerializableFunction<TopicPartition, Boolean>)
+                      fromByteArray(checkStopReadinfFn));
+        }
 
-      byte[] checkStopReadinfFn = configRow.getBytes("check_stop_reading_fn");
-      if (checkStopReadinfFn != null) {
-        transform =
-            transform.withCheckStopReadingFn(
-                (SerializableFunction<TopicPartition, Boolean>) fromByteArray(checkStopReadinfFn));
+        return transform;
+      } catch (InvalidClassException e) {
+        throw new RuntimeException(e);
       }
-
-      return transform;
     }
   }
 
@@ -496,68 +499,83 @@ public Row toConfigRow(Write<?, ?> transform) {
                 });
         fieldValues.put("producer_config", producerConfigMap);
       }
+      if (writeRecordsTransform.getBadRecordErrorHandler() != null
+          && !(writeRecordsTransform.getBadRecordErrorHandler()
+              instanceof ErrorHandler.DefaultErrorHandler)) {
+        throw new RuntimeException(
+            "Upgrading KafkaIO write transforms that have `withBadRecordErrorHandler` property set"
+                + "is not supported yet.");
+      }
 
       return Row.withSchema(schema).withFieldValues(fieldValues).build();
     }
 
     @Override
     public Write<?, ?> fromConfigRow(Row configRow) {
-      Write<?, ?> transform = KafkaIO.write();
+      try {
+        Write<?, ?> transform = KafkaIO.write();
 
-      String bootstrapServers = configRow.getString("bootstrap_servers");
-      if (bootstrapServers != null) {
-        transform = transform.withBootstrapServers(bootstrapServers);
-      }
-      String topic = configRow.getValue("topic");
-      if (topic != null) {
-        transform = transform.withTopic(topic);
-      }
-      byte[] keySerializerBytes = configRow.getBytes("key_serializer");
-      if (keySerializerBytes != null) {
-        transform = transform.withKeySerializer((Class) fromByteArray(keySerializerBytes));
-      }
-      byte[] valueSerializerBytes = configRow.getBytes("value_serializer");
-      if (valueSerializerBytes != null) {
-        transform = transform.withValueSerializer((Class) fromByteArray(valueSerializerBytes));
-      }
-      byte[] producerFactoryFnBytes = configRow.getBytes("producer_factory_fn");
-      if (producerFactoryFnBytes != null) {
-        transform =
-            transform.withProducerFactoryFn(
-                (SerializableFunction) fromByteArray(producerFactoryFnBytes));
-      }
-      Boolean isEOS = configRow.getBoolean("eos");
-      if (isEOS != null && isEOS) {
-        Integer numShards = configRow.getInt32("num_shards");
-        String sinkGroupId = configRow.getString("sink_group_id");
-        if (numShards == null) {
-          throw new IllegalArgumentException(
-              "Expected numShards to be provided when EOS is set to true");
+        String bootstrapServers = configRow.getString("bootstrap_servers");
+        if (bootstrapServers != null) {
+          transform = transform.withBootstrapServers(bootstrapServers);
         }
-        if (sinkGroupId == null) {
-          throw new IllegalArgumentException(
-              "Expected sinkGroupId to be provided when EOS is set to true");
+        String topic = configRow.getValue("topic");
+        if (topic != null) {
+          transform = transform.withTopic(topic);
+        }
+        byte[] keySerializerBytes = configRow.getBytes("key_serializer");
+        if (keySerializerBytes != null) {
+          transform = transform.withKeySerializer((Class) fromByteArray(keySerializerBytes));
+        }
+        byte[] valueSerializerBytes = configRow.getBytes("value_serializer");
+        if (valueSerializerBytes != null) {
+          transform = transform.withValueSerializer((Class) fromByteArray(valueSerializerBytes));
+        }
+        byte[] producerFactoryFnBytes = configRow.getBytes("producer_factory_fn");
+        if (producerFactoryFnBytes != null) {
+          transform =
+              transform.withProducerFactoryFn(
+                  (SerializableFunction) fromByteArray(producerFactoryFnBytes));
+        }
+        Boolean isEOS = configRow.getBoolean("eos");
+        if (isEOS != null && isEOS) {
+          Integer numShards = configRow.getInt32("num_shards");
+          String sinkGroupId = configRow.getString("sink_group_id");
+          if (numShards == null) {
+            throw new IllegalArgumentException(
+                "Expected numShards to be provided when EOS is set to true");
+          }
+          if (sinkGroupId == null) {
+            throw new IllegalArgumentException(
+                "Expected sinkGroupId to be provided when EOS is set to true");
+          }
+          transform = transform.withEOS(numShards, sinkGroupId);
+        }
+        byte[] consumerFactoryFnBytes = configRow.getBytes("consumer_factory_fn");
+        if (consumerFactoryFnBytes != null) {
+          transform =
+              transform.withConsumerFactoryFn(
+                  (SerializableFunction) fromByteArray(consumerFactoryFnBytes));
         }
-        transform = transform.withEOS(numShards, sinkGroupId);
-      }
-      byte[] consumerFactoryFnBytes = configRow.getBytes("consumer_factory_fn");
-      if (consumerFactoryFnBytes != null) {
-        transform =
-            transform.withConsumerFactoryFn(
-                (SerializableFunction) fromByteArray(consumerFactoryFnBytes));
-      }
 
-      Map<String, byte[]> producerConfig = configRow.getMap("producer_config");
-      if (producerConfig != null && !producerConfig.isEmpty()) {
-        Map<String, Object> updatedProducerConfig = new HashMap<>();
-        producerConfig.forEach(
-            (key, dataBytes) -> {
-              updatedProducerConfig.put(key, fromByteArray((byte[]) dataBytes));
-            });
-        transform = transform.withProducerConfigUpdates(updatedProducerConfig);
-      }
+        Map<String, byte[]> producerConfig = configRow.getMap("producer_config");
+        if (producerConfig != null && !producerConfig.isEmpty()) {
+          Map<String, Object> updatedProducerConfig = new HashMap<>();
+          producerConfig.forEach(
+              (key, dataBytes) -> {
+                try {
+                  updatedProducerConfig.put(key, fromByteArray((byte[]) dataBytes));
+                } catch (InvalidClassException e) {
+                  throw new RuntimeException(e);
+                }
+              });
+          transform = transform.withProducerConfigUpdates(updatedProducerConfig);
+        }
 
-      return transform;
+        return transform;
+      } catch (InvalidClassException e) {
+        throw new RuntimeException(e);
+      }
     }
   }
 
diff --git a/sdks/java/io/kafka/upgrade/src/test/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslationTest.java b/sdks/java/io/kafka/upgrade/src/test/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslationTest.java
index d99bee0ad209..be54d7830d5e 100644
--- a/sdks/java/io/kafka/upgrade/src/test/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslationTest.java
+++ b/sdks/java/io/kafka/upgrade/src/test/java/org/apache/beam/sdk/io/kafka/upgrade/KafkaIOTranslationTest.java
@@ -35,6 +35,7 @@
 import org.apache.beam.sdk.io.kafka.upgrade.KafkaIOTranslation.KafkaIOReadWithMetadataTranslator;
 import org.apache.beam.sdk.io.kafka.upgrade.KafkaIOTranslation.KafkaIOWriteTranslator;
 import org.apache.beam.sdk.values.Row;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
 import org.apache.kafka.clients.consumer.ConsumerConfig;
 import org.apache.kafka.common.TopicPartition;
 import org.junit.Test;
@@ -123,6 +124,9 @@ public void testReCreateReadTransformFromRow() throws Exception {
 
   @Test
   public void testReadTransformRowIncludesAllFields() throws Exception {
+    // TODO: support 'withBadRecordErrorHandler' property.
+    List<String> fieldsToIgnore =
+        ImmutableList.of("getBadRecordRouter", "getBadRecordErrorHandler");
     List<String> getMethodNames =
         Arrays.stream(Read.class.getDeclaredMethods())
             .map(
@@ -130,6 +134,7 @@ public void testReadTransformRowIncludesAllFields() throws Exception {
                   return method.getName();
                 })
             .filter(methodName -> methodName.startsWith("get"))
+            .filter(methodName -> !fieldsToIgnore.contains(methodName))
             .collect(Collectors.toList());
 
     // Just to make sure that this does not pass trivially.
@@ -189,6 +194,11 @@ public void testReCreateWriteTransformFromRow() throws Exception {
 
   @Test
   public void testWriteTransformRowIncludesAllFields() throws Exception {
+    // For these fields, default value will suffice (so no need to serialize when upgrading).
+    // TODO: support 'withBadRecordErrorHandler' property.
+    List<String> fieldsToIgnore =
+        ImmutableList.of("getBadRecordRouter", "getBadRecordErrorHandler");
+
     // Write transform delegates property handling to the WriteRecords class. So we inspect the
     // WriteRecords class here.
     List<String> getMethodNames =
@@ -198,6 +208,7 @@ public void testWriteTransformRowIncludesAllFields() throws Exception {
                   return method.getName();
                 })
             .filter(methodName -> methodName.startsWith("get"))
+            .filter(methodName -> !fieldsToIgnore.contains(methodName))
             .collect(Collectors.toList());
 
     // Just to make sure that this does not pass trivially.
diff --git a/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java b/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java
index 268c6b088d89..1dd95b44c2fc 100644
--- a/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java
+++ b/sdks/java/io/neo4j/src/main/java/org/apache/beam/sdk/io/neo4j/Neo4jIO.java
@@ -814,17 +814,19 @@ private void executeReadCypherStatement(
       // We could actually read and write here depending on the type of transaction
       // we picked.  As long as the Cypher statement returns values it's fine.
       //
-      TransactionWork<List<OutputT>> transactionWork =
+      TransactionWork<Long> transactionWork =
           transaction -> {
+            long count = 0L;
             Result result = transaction.run(cypher, parametersMap);
-            return result.list(
-                record -> {
-                  try {
-                    return rowMapper.mapRow(record);
-                  } catch (Exception e) {
-                    throw new RuntimeException("error mapping Neo4j record to row", e);
-                  }
-                });
+            while (result.hasNext()) {
+              try {
+                processContext.output(rowMapper.mapRow(result.next()));
+              } catch (Exception e) {
+                throw new RuntimeException("Error mapping Neo4J result", e);
+              }
+              count++;
+            }
+            return count;
           };
 
       if (logCypher) {
@@ -846,13 +848,13 @@ record -> {
       if (driverSession.session == null) {
         throw new RuntimeException("neo4j session was not initialized correctly");
       } else {
-        List<OutputT> outputs;
+        final Long count;
         if (writeTransaction) {
-          outputs = driverSession.session.writeTransaction(transactionWork, transactionConfig);
+          count = driverSession.session.writeTransaction(transactionWork, transactionConfig);
         } else {
-          outputs = driverSession.session.readTransaction(transactionWork, transactionConfig);
+          count = driverSession.session.readTransaction(transactionWork, transactionConfig);
         }
-        outputs.forEach(processContext::output);
+        LOG.debug("Retrieved " + count + " elements from Neo4J");
       }
     }
   }
diff --git a/sdks/java/io/pulsar/src/test/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFnTest.java b/sdks/java/io/pulsar/src/test/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFnTest.java
index 273a1915d2bb..b72fe423efb1 100644
--- a/sdks/java/io/pulsar/src/test/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFnTest.java
+++ b/sdks/java/io/pulsar/src/test/java/org/apache/beam/sdk/io/pulsar/ReadFromPulsarDoFnTest.java
@@ -21,11 +21,14 @@
 import static org.junit.Assert.assertTrue;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import org.apache.beam.sdk.io.range.OffsetRange;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.apache.pulsar.client.api.MessageId;
 import org.apache.pulsar.client.api.PulsarClient;
 import org.apache.pulsar.client.internal.DefaultImplementation;
@@ -173,6 +176,16 @@ public void outputWithTimestamp(
       records.add(output);
     }
 
+    @Override
+    public void outputWindowedValue(
+        PulsarMessage output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      throw new UnsupportedOperationException(
+          "unsupported outputWindowedValue in mock outputreceiver");
+    }
+
     public List<PulsarMessage> getOutputs() {
       return records;
     }
diff --git a/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Repeater.java b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Repeater.java
new file mode 100644
index 000000000000..e298511494a1
--- /dev/null
+++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/Repeater.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.io.requestresponse;
+
+import com.google.auto.value.AutoValue;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.util.BackOff;
+import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.Sleeper;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableSet;
+
+/**
+ * Repeats a method invocation when it encounters an error, pausing invocations using {@link
+ * Sleeper} for a {@link BackOff#nextBackOffMillis}.
+ */
+@AutoValue
+abstract class Repeater<InputT, OutputT> {
+
+  /** {@link Set} of {@link UserCodeExecutionException}s that warrant repeating. */
+  static final Set<Class<? extends UserCodeExecutionException>> REPEATABLE_ERROR_TYPES =
+      ImmutableSet.of(
+          UserCodeRemoteSystemException.class,
+          UserCodeTimeoutException.class,
+          UserCodeQuotaException.class);
+
+  static <InputT, OutputT> Builder<InputT, OutputT> builder() {
+    return new AutoValue_Repeater.Builder<>();
+  }
+
+  /**
+   * The {@link ThrowableFunction} to invoke repeatedly until it succeeds, throws a {@link
+   * UserCodeExecutionException} that is not {@link #REPEATABLE_ERROR_TYPES}, or {@link
+   * BackOff#STOP}.
+   */
+  abstract ThrowableFunction<InputT, OutputT> getThrowableFunction();
+
+  /**
+   * The {@link Sleeper} that pauses execution of the {@link #getThrowableFunction} when it throws a
+   * {@link #REPEATABLE_ERROR_TYPES} {@link UserCodeExecutionException}. Uses {@link
+   * Sleeper#DEFAULT} by default.
+   */
+  abstract Sleeper getSleeper();
+
+  /**
+   * The {@link BackOff} that reports to {@link #getSleeper} how long to pause execution. It reports
+   * a {@link BackOff#STOP} to stop repeating invocation attempts. Uses {@link
+   * FluentBackoff#DEFAULT#getBackOff} by default.
+   */
+  abstract BackOff getBackOff();
+
+  /**
+   * Applies the {@link InputT} to the {@link ThrowableFunction}, returning the {@link OutputT} if
+   * successful. If the function throws an exception that {@link #REPEATABLE_ERROR_TYPES} contains,
+   * repeats the invocation after {@link Sleeper#sleep} for the amount of time reported by {@link
+   * BackOff#nextBackOffMillis}. Throws the latest encountered {@link UserCodeExecutionException}
+   * when {@link BackOff} reports a {@link BackOff#STOP}.
+   */
+  OutputT apply(InputT input) throws UserCodeExecutionException {
+    Optional<UserCodeExecutionException> latestError = Optional.empty();
+    long waitFor = 0L;
+    while (waitFor != BackOff.STOP) {
+      try {
+        getSleeper().sleep(waitFor);
+        return getThrowableFunction().apply(input);
+      } catch (UserCodeExecutionException e) {
+        if (!REPEATABLE_ERROR_TYPES.contains(e.getClass())) {
+          throw e;
+        }
+        latestError = Optional.of(e);
+      } catch (InterruptedException ignored) {
+      }
+      try {
+        waitFor = getBackOff().nextBackOffMillis();
+      } catch (IOException e) {
+        throw new UserCodeExecutionException(e);
+      }
+    }
+    throw latestError.orElse(
+        new UserCodeExecutionException("failed to process for input: " + input));
+  }
+
+  /**
+   * A {@link FunctionalInterface} for executing a {@link UserCodeExecutionException} throwable
+   * function.
+   */
+  @FunctionalInterface
+  interface ThrowableFunction<InputT, OutputT> {
+    /** Returns the result of invoking this function on the given input. */
+    OutputT apply(InputT input) throws UserCodeExecutionException;
+  }
+
+  @AutoValue.Builder
+  abstract static class Builder<InputT, OutputT> {
+
+    /** See {@link #getThrowableFunction}. */
+    abstract Builder<InputT, OutputT> setThrowableFunction(
+        ThrowableFunction<InputT, OutputT> value);
+
+    /** See {@link #getSleeper}. */
+    abstract Builder<InputT, OutputT> setSleeper(Sleeper value);
+
+    abstract Optional<Sleeper> getSleeper();
+
+    /** See {@link #getBackOff}. */
+    abstract Builder<InputT, OutputT> setBackOff(BackOff value);
+
+    abstract Optional<BackOff> getBackOff();
+
+    abstract Repeater<InputT, OutputT> autoBuild();
+
+    final Repeater<InputT, OutputT> build() {
+      if (!getSleeper().isPresent()) {
+        setSleeper(Sleeper.DEFAULT);
+      }
+
+      if (!getBackOff().isPresent()) {
+        setBackOff(FluentBackoff.DEFAULT.backoff());
+      }
+
+      return autoBuild();
+    }
+  }
+}
diff --git a/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Samza.groovy b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeRemoteSystemException.java
similarity index 51%
rename from .test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Samza.groovy
rename to sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeRemoteSystemException.java
index 891f25a7e12b..dac16344bec7 100644
--- a/.test-infra/jenkins/job_PostCommit_Go_ValidatesRunner_Samza.groovy
+++ b/sdks/java/io/rrio/src/main/java/org/apache/beam/io/requestresponse/UserCodeRemoteSystemException.java
@@ -15,24 +15,27 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.beam.io.requestresponse;
 
-import CommonJobProperties as commonJobProperties
-import PostcommitJobBuilder
+/**
+ * A {@link UserCodeExecutionException} that signals an error with a remote system. Examples of such
+ * errors include an HTTP 5XX error or gRPC INTERNAL (13) error.
+ */
+public class UserCodeRemoteSystemException extends UserCodeExecutionException {
+  public UserCodeRemoteSystemException(String message) {
+    super(message);
+  }
 
-// This job runs the suite of Go integration tests against the Samza runner.
-PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go_VR_Samza',
-    'Run Go Samza ValidatesRunner', 'Go Samza ValidatesRunner Tests', this) {
-      description('Runs Go integration tests on the Samza runner.')
+  public UserCodeRemoteSystemException(String message, Throwable cause) {
+    super(message, cause);
+  }
 
-      // Set common parameters.
-      commonJobProperties.setTopLevelMainJobProperties(delegate)
+  public UserCodeRemoteSystemException(Throwable cause) {
+    super(cause);
+  }
 
-      // Gradle goals for this job.
-      steps {
-        gradle {
-          rootBuildScriptDir(commonJobProperties.checkoutDir)
-          tasks(':sdks:go:test:samzaValidatesRunner')
-          commonJobProperties.setGradleSwitches(delegate)
-        }
-      }
-    }
+  public UserCodeRemoteSystemException(
+      String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+  }
+}
diff --git a/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/RepeaterTest.java b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/RepeaterTest.java
new file mode 100644
index 000000000000..14bb25ce4ad4
--- /dev/null
+++ b/sdks/java/io/rrio/src/test/java/org/apache/beam/io/requestresponse/RepeaterTest.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.io.requestresponse;
+
+import static org.apache.beam.io.requestresponse.Repeater.REPEATABLE_ERROR_TYPES;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertThrows;
+
+import java.lang.reflect.InvocationTargetException;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.util.FluentBackoff;
+import org.apache.beam.sdk.util.Sleeper;
+import org.apache.beam.sdk.util.UserCodeException;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.util.concurrent.UncheckedExecutionException;
+import org.junit.Rule;
+import org.junit.Test;
+
+public class RepeaterTest {
+
+  @Rule public TestPipeline pipeline = TestPipeline.create();
+
+  private static final TupleTag<Integer> OUTPUT_TAG = new TupleTag<Integer>() {};
+  private static final TupleTag<String> FAILURE_TAG = new TupleTag<String>() {};
+
+  private static final int LIMIT = 3;
+  private static final FluentBackoff FLUENT_BACKOFF = FluentBackoff.DEFAULT.withMaxRetries(LIMIT);
+
+  @Test
+  public void givenCallerQuotaErrorsExceedsLimit_emitsIntoFailurePCollection() {
+    PCollectionTuple pct =
+        pipeline
+            .apply(Create.of(1))
+            .apply(
+                ParDo.of(
+                        new DoFnWithRepeaters(
+                            new CallerImpl(LIMIT + 1, UserCodeQuotaException.class),
+                            new SetupTeardownImpl(0)))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    PAssert.that(pct.get(OUTPUT_TAG)).empty();
+    PAssert.that(pct.get(FAILURE_TAG)).containsInAnyOrder(UserCodeQuotaException.class.getName());
+
+    pipeline.run();
+  }
+
+  @Test
+  public void givenSetupQuotaErrorsExceedsLimit_throws() {
+    pipeline
+        .apply(Create.of(1))
+        .apply(
+            ParDo.of(
+                    new DoFnWithRepeaters(
+                        new CallerImpl(0),
+                        new SetupTeardownImpl(LIMIT + 1, UserCodeQuotaException.class)))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    UncheckedExecutionException thrown =
+        assertThrows(UncheckedExecutionException.class, pipeline::run);
+    assertThat(thrown.getCause(), allOf(notNullValue(), instanceOf(UserCodeException.class)));
+    assertThat(
+        thrown.getCause().getCause(),
+        allOf(notNullValue(), instanceOf(UserCodeQuotaException.class)));
+  }
+
+  @Test
+  public void givenCallerTimeoutErrorsExceedsLimit_emitsIntoFailurePCollection() {
+    PCollectionTuple pct =
+        pipeline
+            .apply(Create.of(1))
+            .apply(
+                ParDo.of(
+                        new DoFnWithRepeaters(
+                            new CallerImpl(LIMIT + 1, UserCodeTimeoutException.class),
+                            new SetupTeardownImpl(0)))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    PAssert.that(pct.get(OUTPUT_TAG)).empty();
+    PAssert.that(pct.get(FAILURE_TAG)).containsInAnyOrder(UserCodeTimeoutException.class.getName());
+
+    pipeline.run();
+  }
+
+  @Test
+  public void givenSetupTimeoutErrorsExceedsLimit_throws() {
+    pipeline
+        .apply(Create.of(1))
+        .apply(
+            ParDo.of(
+                    new DoFnWithRepeaters(
+                        new CallerImpl(0),
+                        new SetupTeardownImpl(LIMIT + 1, UserCodeTimeoutException.class)))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    UncheckedExecutionException thrown =
+        assertThrows(UncheckedExecutionException.class, pipeline::run);
+    assertThat(thrown.getCause(), allOf(notNullValue(), instanceOf(UserCodeException.class)));
+    assertThat(
+        thrown.getCause().getCause(),
+        allOf(notNullValue(), instanceOf(UserCodeTimeoutException.class)));
+  }
+
+  @Test
+  public void givenCallerRemoteSystemExceptionExceedsLimit_emitsIntoFailurePCollection() {
+    PCollectionTuple pct =
+        pipeline
+            .apply(Create.of(1))
+            .apply(
+                ParDo.of(
+                        new DoFnWithRepeaters(
+                            new CallerImpl(LIMIT + 1, UserCodeRemoteSystemException.class),
+                            new SetupTeardownImpl(0)))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    PAssert.that(pct.get(OUTPUT_TAG)).empty();
+    PAssert.that(pct.get(FAILURE_TAG))
+        .containsInAnyOrder(UserCodeRemoteSystemException.class.getName());
+
+    pipeline.run();
+  }
+
+  @Test
+  public void givenSetupRemoteSystemErrorsExceedsLimit_throws() {
+    pipeline
+        .apply(Create.of(1))
+        .apply(
+            ParDo.of(
+                    new DoFnWithRepeaters(
+                        new CallerImpl(0),
+                        new SetupTeardownImpl(LIMIT + 1, UserCodeRemoteSystemException.class)))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    UncheckedExecutionException thrown =
+        assertThrows(UncheckedExecutionException.class, pipeline::run);
+    assertThat(thrown.getCause(), allOf(notNullValue(), instanceOf(UserCodeException.class)));
+    assertThat(
+        thrown.getCause().getCause(),
+        allOf(notNullValue(), instanceOf(UserCodeRemoteSystemException.class)));
+  }
+
+  @Test
+  public void givenCallerNonRepeatableError_emitsIntoFailurePCollection() {
+    PCollectionTuple pct =
+        pipeline
+            .apply(Create.of(1))
+            .apply(
+                ParDo.of(
+                        new DoFnWithRepeaters(
+                            new CallerImpl(1, UserCodeExecutionException.class),
+                            new SetupTeardownImpl(0)))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    PAssert.that(pct.get(OUTPUT_TAG)).empty();
+    PAssert.that(pct.get(FAILURE_TAG))
+        .containsInAnyOrder(UserCodeExecutionException.class.getName());
+
+    pipeline.run();
+  }
+
+  @Test
+  public void givenSetupNonRepeatableError_throws() {
+    pipeline
+        .apply(Create.of(1))
+        .apply(
+            ParDo.of(
+                    new DoFnWithRepeaters(
+                        new CallerImpl(0),
+                        new SetupTeardownImpl(1, UserCodeExecutionException.class)))
+                .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    UncheckedExecutionException thrown =
+        assertThrows(UncheckedExecutionException.class, pipeline::run);
+    assertThat(thrown.getCause(), allOf(notNullValue(), instanceOf(UserCodeException.class)));
+    assertThat(
+        thrown.getCause().getCause(),
+        allOf(notNullValue(), instanceOf(UserCodeExecutionException.class)));
+  }
+
+  @Test
+  public void givenRepeatableErrorBelowLimit_emitsIntoOutputPCollection() {
+    PCollectionTuple pct =
+        pipeline
+            .apply(Create.of(1))
+            .apply(
+                ParDo.of(
+                        new DoFnWithRepeaters(
+                            new CallerImpl(LIMIT - 1, UserCodeQuotaException.class),
+                            new SetupTeardownImpl(0)))
+                    .withOutputTags(OUTPUT_TAG, TupleTagList.of(FAILURE_TAG)));
+
+    PAssert.that(pct.get(OUTPUT_TAG)).containsInAnyOrder(2);
+    PAssert.that(pct.get(FAILURE_TAG)).empty();
+
+    pipeline.run();
+  }
+
+  private static class DoFnWithRepeaters extends DoFn<Integer, Integer> {
+    private final CallerImpl caller;
+    private final SetupTeardownImpl setupTeardown;
+
+    private DoFnWithRepeaters(CallerImpl caller, SetupTeardownImpl setupTeardown) {
+      this.caller = caller;
+      this.setupTeardown = setupTeardown;
+    }
+
+    @Setup
+    public void setup() throws UserCodeExecutionException {
+      Repeater<Void, Void> repeater =
+          Repeater.<Void, Void>builder()
+              .setSleeper(new NoOpSleeper())
+              .setBackOff(FLUENT_BACKOFF.backoff())
+              .setThrowableFunction(
+                  input -> {
+                    setupTeardown.setup();
+                    return null;
+                  })
+              .build();
+      repeater.apply(null);
+    }
+
+    @ProcessElement
+    public void process(@Element Integer element, MultiOutputReceiver receiver) {
+      Repeater<Integer, Integer> repeater =
+          Repeater.<Integer, Integer>builder()
+              .setSleeper(new NoOpSleeper())
+              .setBackOff(FLUENT_BACKOFF.backoff())
+              .setThrowableFunction(caller::call)
+              .build();
+      try {
+        Integer output = repeater.apply(element);
+        receiver.get(OUTPUT_TAG).output(output);
+      } catch (UserCodeExecutionException e) {
+        receiver.get(FAILURE_TAG).output(getRepeatableErrorTypeName(e.getClass()));
+      }
+    }
+  }
+
+  private static class CallerImpl implements Caller<Integer, Integer> {
+
+    private int wantNumErrors;
+    private final Class<? extends UserCodeExecutionException> wantThrowWith;
+    private final String exceptionName;
+
+    private CallerImpl(int wantNumErrors) {
+      this(wantNumErrors, UserCodeExecutionException.class);
+    }
+
+    private CallerImpl(
+        int wantNumErrors, Class<? extends UserCodeExecutionException> wantThrowWith) {
+      this.wantNumErrors = wantNumErrors;
+      this.wantThrowWith = wantThrowWith;
+      this.exceptionName = getRepeatableErrorTypeName(wantThrowWith);
+    }
+
+    @Override
+    public Integer call(Integer request) throws UserCodeExecutionException {
+      if (wantNumErrors > 0) {
+        wantNumErrors--;
+        try {
+          throw wantThrowWith.getConstructor(String.class).newInstance(exceptionName);
+        } catch (InstantiationException
+            | NoSuchMethodException
+            | InvocationTargetException
+            | IllegalAccessException e) {
+          throw new RuntimeException(e);
+        }
+      }
+      return request * 2;
+    }
+  }
+
+  private static class SetupTeardownImpl implements SetupTeardown {
+    private int wantNumErrors;
+    private final Class<? extends UserCodeExecutionException> wantThrowWith;
+    private final String exceptionName;
+
+    private SetupTeardownImpl(int wantNumErrors) {
+      this(wantNumErrors, UserCodeExecutionException.class);
+    }
+
+    private SetupTeardownImpl(
+        int wantNumErrors, Class<? extends UserCodeExecutionException> wantThrowWith) {
+      this.wantNumErrors = wantNumErrors;
+      this.wantThrowWith = wantThrowWith;
+      this.exceptionName = getRepeatableErrorTypeName(wantThrowWith);
+    }
+
+    @Override
+    public void setup() throws UserCodeExecutionException {
+      if (wantNumErrors > 0) {
+        wantNumErrors--;
+        try {
+          throw wantThrowWith.getConstructor(String.class).newInstance(exceptionName);
+        } catch (InstantiationException
+            | NoSuchMethodException
+            | InvocationTargetException
+            | IllegalAccessException e) {
+          throw new RuntimeException(e);
+        }
+      }
+    }
+
+    @Override
+    public void teardown() throws UserCodeExecutionException {}
+  }
+
+  private static String getRepeatableErrorTypeName(Class<? extends UserCodeExecutionException> e) {
+    for (Class<? extends UserCodeExecutionException> ex : REPEATABLE_ERROR_TYPES) {
+      if (ex.equals(e)) {
+        return ex.getName();
+      }
+    }
+    return e.getName();
+  }
+
+  // We don't want to wait for sleep to delay test invocations.
+  private static class NoOpSleeper implements Sleeper {
+    @Override
+    public void sleep(long millis) throws InterruptedException {}
+  }
+}
diff --git a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReadFromSparkReceiverWithOffsetDoFnTest.java b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReadFromSparkReceiverWithOffsetDoFnTest.java
index 33827164c6b7..bb0e6524241d 100644
--- a/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReadFromSparkReceiverWithOffsetDoFnTest.java
+++ b/sdks/java/io/sparkreceiver/2/src/test/java/org/apache/beam/sdk/io/sparkreceiver/ReadFromSparkReceiverWithOffsetDoFnTest.java
@@ -22,12 +22,15 @@
 import static org.junit.Assert.assertTrue;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import org.apache.beam.sdk.io.range.OffsetRange;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
 import org.apache.beam.sdk.transforms.splittabledofn.OffsetRangeTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.SplitResult;
+import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.transforms.windowing.PaneInfo;
 import org.checkerframework.checker.initialization.qual.Initialized;
 import org.checkerframework.checker.nullness.qual.NonNull;
 import org.checkerframework.checker.nullness.qual.UnknownKeyFor;
@@ -64,6 +67,16 @@ public void outputWithTimestamp(
       records.add(output);
     }
 
+    @Override
+    public void outputWindowedValue(
+        String output,
+        Instant timestamp,
+        Collection<? extends BoundedWindow> windows,
+        PaneInfo paneInfo) {
+      throw new UnsupportedOperationException(
+          "Not expecting to receive call to outputWindowedValue");
+    }
+
     public List<String> getOutputs() {
       return this.records;
     }
diff --git a/sdks/python/apache_beam/examples/inference/README.md b/sdks/python/apache_beam/examples/inference/README.md
index cd92d9c127ee..3bb68440ed60 100644
--- a/sdks/python/apache_beam/examples/inference/README.md
+++ b/sdks/python/apache_beam/examples/inference/README.md
@@ -113,7 +113,7 @@ pip install skl2onnx
 
 ### Additional resources
 For more information, see the
-[Machine Learning](/documentation/sdks/python-machine-learning) and the
+[About Beam ML](/documentation/ml/about-ml) and the
 [RunInference transform](/documentation/transforms/python/elementwise/runinference) documentation.
 
 ---
diff --git a/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py b/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py
index 15a68228c661..72a65ab72810 100644
--- a/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py
+++ b/sdks/python/apache_beam/examples/inference/anomaly_detection/anomaly_detection_pipeline/setup.py
@@ -32,7 +32,7 @@
     "apache-beam[gcp]==2.41.0",
     "hdbscan==0.8.28",
     "scikit-learn==1.0.2",
-    "transformers==4.30.0",
+    "transformers==4.36.0",
     "torch==1.13.1",
     "pandas==1.3.5",
     "yagmail==0.15.283",
diff --git a/sdks/python/apache_beam/examples/inference/large_language_modeling/requirements.txt b/sdks/python/apache_beam/examples/inference/large_language_modeling/requirements.txt
index aaec7156dc7b..b5383f964b2f 100644
--- a/sdks/python/apache_beam/examples/inference/large_language_modeling/requirements.txt
+++ b/sdks/python/apache_beam/examples/inference/large_language_modeling/requirements.txt
@@ -17,5 +17,5 @@
 # under the License
 
 torch==1.13.1
-transformers==4.30.0
+transformers==4.36.0
 sentencepiece==0.1.97
\ No newline at end of file
diff --git a/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py b/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py
index 102699b6bec9..d1bc0a06c4b2 100644
--- a/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py
+++ b/sdks/python/apache_beam/examples/inference/runinference_metrics/setup.py
@@ -29,7 +29,7 @@
 from setuptools import find_packages
 
 REQUIREMENTS = [
-    "apache-beam[gcp]==2.41.0", "transformers==4.30.0", "torch==1.13.1"
+    "apache-beam[gcp]==2.41.0", "transformers==4.36.0", "torch==1.13.1"
 ]
 
 setuptools.setup(
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py
index 0db10718295b..261b480b1083 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py
@@ -57,6 +57,7 @@ def check_mltransform_compute_and_apply_vocabulary_with_scalar():
 Row(x=array([4]))
 Row(x=array([1]))
 Row(x=array([0]))
+Row(x=array([0]))
 Row(x=array([2]))
 Row(x=array([3]))
   [END mltransform_compute_and_apply_vocabulary_with_scalar] '''.splitlines(
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index 9225acf346e4..24df59ddc5cc 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -75,8 +75,15 @@
 
 
 class ReadFromAvro(PTransform):
-  """A :class:`~apache_beam.transforms.ptransform.PTransform` for reading avro
-  files."""
+  """A `PTransform` for reading records from avro files.
+
+  Each record of the resulting PCollection will contain
+  a single record read from a source. Records that are of simple types will be
+  mapped to beam Rows with a single `record` field containing the records
+  value. Records that are of Avro type ``RECORD`` will be mapped to Beam rows
+  that comply with the schema contained in the Avro file that contains those
+  records.
+  """
   def __init__(
       self,
       file_pattern=None,
@@ -355,7 +362,10 @@ def split_points_unclaimed(stop_position):
 
 
 class WriteToAvro(beam.transforms.PTransform):
-  """A ``PTransform`` for writing avro files."""
+  """A ``PTransform`` for writing avro files.
+
+  If the input has a schema, a corresponding avro schema will be automatically
+  generated and used to write the output records."""
   def __init__(
       self,
       file_path_prefix,
diff --git a/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py b/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py
index a2f350e8cb7a..a7bf686d0642 100644
--- a/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py
+++ b/sdks/python/apache_beam/io/external/xlang_kafkaio_it_test.py
@@ -143,7 +143,7 @@ def test_local_kafkaio_null_key(self):
       self.run_kafka_write(pipeline_creator)
       self.run_kafka_read(pipeline_creator, None)
 
-  @pytest.mark.uses_io_expansion_service
+  @pytest.mark.uses_io_java_expansion_service
   @unittest.skipUnless(
       os.environ.get('EXPANSION_PORT'),
       "EXPANSION_PORT environment var is not provided.")
@@ -162,7 +162,7 @@ def test_hosted_kafkaio_populated_key(self):
     self.run_kafka_write(pipeline_creator)
     self.run_kafka_read(pipeline_creator, b'key')
 
-  @pytest.mark.uses_io_expansion_service
+  @pytest.mark.uses_io_java_expansion_service
   @unittest.skipUnless(
       os.environ.get('EXPANSION_PORT'),
       "EXPANSION_PORT environment var is not provided.")
diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py
index 184138af7525..ac06425e95a9 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery.py
@@ -72,7 +72,8 @@
 or a table. Pipeline construction will fail with a validation error if neither
 or both are specified.
 
-When reading via `ReadFromBigQuery`, bytes are returned decoded as bytes.
+When reading via `ReadFromBigQuery` using `EXPORT`,
+bytes are returned decoded as bytes.
 This is due to the fact that ReadFromBigQuery uses Avro exports by default.
 When reading from BigQuery using `apache_beam.io.BigQuerySource`, bytes are
 returned as base64-encoded bytes. To get base64-encoded bytes using
@@ -2597,6 +2598,8 @@ def expand(self, input):
 
 
 class ReadFromBigQuery(PTransform):
+  # pylint: disable=line-too-long,W1401
+
   """Read data from BigQuery.
 
     This PTransform uses a BigQuery export job to take a snapshot of the table
@@ -2653,8 +2656,7 @@ class ReadFromBigQuery(PTransform):
       :data:`None`, then the temp_location parameter is used.
     bigquery_job_labels (dict): A dictionary with string labels to be passed
       to BigQuery export and query jobs created by this transform. See:
-      https://cloud.google.com/bigquery/docs/reference/rest/v2/\
-              Job#JobConfiguration
+      https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfiguration
     use_json_exports (bool): By default, this transform works by exporting
       BigQuery data into Avro files, and reading those files. With this
       parameter, the transform will instead export to JSON files. JSON files
@@ -2666,11 +2668,10 @@ class ReadFromBigQuery(PTransform):
       types (datetime.date, datetime.datetime, datetime.datetime,
       and datetime.datetime respectively). Avro exports are recommended.
       To learn more about BigQuery types, and Time-related type
-      representations, see: https://cloud.google.com/bigquery/docs/reference/\
-              standard-sql/data-types
+      representations,
+      see: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
       To learn more about type conversions between BigQuery and Avro, see:
-      https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-avro\
-              #avro_conversions
+      https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-avro\#avro_conversions
     temp_dataset (``apache_beam.io.gcp.internal.clients.bigquery.\
         DatasetReference``):
         Temporary dataset reference to use when reading from BigQuery using a
@@ -2690,8 +2691,7 @@ class ReadFromBigQuery(PTransform):
       (`PYTHON_DICT`). There is experimental support for producing a
       PCollection with a schema and yielding Beam Rows via the option
       `BEAM_ROW`. For more information on schemas, see
-      https://beam.apache.org/documentation/programming-guide/\
-      #what-is-a-schema)
+      https://beam.apache.org/documentation/programming-guide/#what-is-a-schema)
       """
   class Method(object):
     EXPORT = 'EXPORT'  #  This is currently the default.
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_test.py b/sdks/python/apache_beam/io/gcp/bigquery_test.py
index 035edffc03f7..e53204a5ebc6 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_test.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_test.py
@@ -43,6 +43,7 @@
 from apache_beam.internal import pickler
 from apache_beam.internal.gcp.json_value import to_json_value
 from apache_beam.io.filebasedsink_test import _TestCaseWithTempDirCleanUp
+from apache_beam.io.filesystems import FileSystems
 from apache_beam.io.gcp import bigquery as beam_bq
 from apache_beam.io.gcp import bigquery_tools
 from apache_beam.io.gcp.bigquery import ReadFromBigQuery
@@ -82,11 +83,13 @@
 try:
   from apache_beam.io.gcp.internal.clients.bigquery import bigquery_v2_client
   from apitools.base.py.exceptions import HttpError
+  from apitools.base.py.exceptions import HttpForbiddenError
   from google.cloud import bigquery as gcp_bigquery
   from google.api_core import exceptions
 except ImportError:
   gcp_bigquery = None
   HttpError = None
+  HttpForbiddenError = None
   exceptions = None
 # pylint: enable=wrong-import-order, wrong-import-position
 
@@ -323,7 +326,9 @@ def test_repeatable_field_is_properly_converted(self):
     self.assertEqual(expected_row, actual)
 
 
-@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed')
+@unittest.skipIf(
+    HttpError is None or HttpForbiddenError is None,
+    'GCP dependencies are not installed')
 class TestReadFromBigQuery(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
@@ -454,6 +459,200 @@ def test_create_temp_dataset_exception(self, exception_type, error_message):
     mock_insert.assert_called()
     self.assertIn(error_message, exc.exception.args[0])
 
+  @parameterized.expand([
+      # first attempt returns a Http 500 blank error and retries
+      # second attempt returns a Http 408 blank error and retries,
+      # third attempt passes
+      param(
+          responses=[
+              HttpForbiddenError(
+                  response={'status': 500}, content="something", url="")
+              if HttpForbiddenError else None,
+              HttpForbiddenError(
+                  response={'status': 408}, content="blank", url="")
+              if HttpForbiddenError else None
+          ],
+          expected_retries=2),
+      # first attempts returns a 403 rateLimitExceeded error
+      # second attempt returns a 429 blank error
+      # third attempt returns a Http 403 rateLimitExceeded error
+      # fourth attempt passes
+      param(
+          responses=[
+              exceptions.Forbidden(
+                  "some message",
+                  errors=({
+                      "message": "transient", "reason": "rateLimitExceeded"
+                  }, )) if exceptions else None,
+              exceptions.ResourceExhausted("some message")
+              if exceptions else None,
+              HttpForbiddenError(
+                  response={'status': 403},
+                  content={
+                      "error": {
+                          "errors": [{
+                              "message": "transient",
+                              "reason": "rateLimitExceeded"
+                          }]
+                      }
+                  },
+                  url="") if HttpForbiddenError else None,
+          ],
+          expected_retries=3),
+  ])
+  def test_get_table_transient_exception(self, responses, expected_retries):
+    class DummyTable:
+      class DummySchema:
+        fields = []
+
+      numBytes = 5
+      schema = DummySchema()
+
+    with mock.patch('time.sleep'), \
+            mock.patch.object(bigquery_v2_client.BigqueryV2.TablesService,
+                              'Get') as mock_get_table, \
+            mock.patch.object(BigQueryWrapper,
+                              'wait_for_bq_job'), \
+            mock.patch.object(BigQueryWrapper,
+                              'perform_extract_job'), \
+            mock.patch.object(FileSystems,
+                              'match'), \
+            mock.patch.object(FileSystems,
+                              'delete'), \
+            beam.Pipeline() as p:
+      call_counter = 0
+
+      def store_callback(unused_request):
+        nonlocal call_counter
+        if call_counter < len(responses):
+          exception = responses[call_counter]
+          call_counter += 1
+          raise exception
+        else:
+          call_counter += 1
+          return DummyTable()
+
+      mock_get_table.side_effect = store_callback
+      _ = p | beam.io.ReadFromBigQuery(
+          table="project.dataset.table", gcs_location="gs://some_bucket")
+
+    # ReadFromBigQuery export mode calls get_table() twice. Once to get
+    # metadata (numBytes), and once to retrieve the table's schema
+    # Any additional calls are retries
+    self.assertEqual(expected_retries, mock_get_table.call_count - 2)
+
+  @parameterized.expand([
+      # first attempt returns a Http 429 with transient reason and retries
+      # second attempt returns a Http 403 with non-transient reason and fails
+      param(
+          responses=[
+              HttpForbiddenError(
+                  response={'status': 429},
+                  content={
+                      "error": {
+                          "errors": [{
+                              "message": "transient",
+                              "reason": "rateLimitExceeded"
+                          }]
+                      }
+                  },
+                  url="") if HttpForbiddenError else None,
+              HttpForbiddenError(
+                  response={'status': 403},
+                  content={
+                      "error": {
+                          "errors": [{
+                              "message": "transient", "reason": "accessDenied"
+                          }]
+                      }
+                  },
+                  url="") if HttpForbiddenError else None
+          ],
+          expected_retries=1),
+      # first attempt returns a transient 403 error and retries
+      # second attempt returns a 403 error with bad contents and fails
+      param(
+          responses=[
+              HttpForbiddenError(
+                  response={'status': 403},
+                  content={
+                      "error": {
+                          "errors": [{
+                              "message": "transient",
+                              "reason": "rateLimitExceeded"
+                          }]
+                      }
+                  },
+                  url="") if HttpForbiddenError else None,
+              HttpError(
+                  response={'status': 403}, content="bad contents", url="")
+              if HttpError else None
+          ],
+          expected_retries=1),
+      # first attempt returns a transient 403 error and retries
+      # second attempt returns a 429 error and retries
+      # third attempt returns a 403 with non-transient reason and fails
+      param(
+          responses=[
+              exceptions.Forbidden(
+                  "some error",
+                  errors=({
+                      "message": "transient", "reason": "rateLimitExceeded"
+                  }, )) if exceptions else None,
+              exceptions.ResourceExhausted("some transient error")
+              if exceptions else None,
+              exceptions.Forbidden(
+                  "some error",
+                  errors=({
+                      "message": "transient", "reason": "accessDenied"
+                  }, )) if exceptions else None,
+          ],
+          expected_retries=2),
+  ])
+  def test_get_table_non_transient_exception(self, responses, expected_retries):
+    class DummyTable:
+      class DummySchema:
+        fields = []
+
+      numBytes = 5
+      schema = DummySchema()
+
+    with mock.patch('time.sleep'), \
+            mock.patch.object(bigquery_v2_client.BigqueryV2.TablesService,
+                              'Get') as mock_get_table, \
+            mock.patch.object(BigQueryWrapper,
+                              'wait_for_bq_job'), \
+            mock.patch.object(BigQueryWrapper,
+                              'perform_extract_job'), \
+            mock.patch.object(FileSystems,
+                              'match'), \
+            mock.patch.object(FileSystems,
+                              'delete'), \
+            self.assertRaises(Exception), \
+            beam.Pipeline() as p:
+      call_counter = 0
+
+      def store_callback(unused_request):
+        nonlocal call_counter
+        if call_counter < len(responses):
+          exception = responses[call_counter]
+          call_counter += 1
+          raise exception
+        else:
+          call_counter += 1
+          return DummyTable()
+
+      mock_get_table.side_effect = store_callback
+      _ = p | beam.io.ReadFromBigQuery(
+          table="project.dataset.table", gcs_location="gs://some_bucket")
+
+    # ReadFromBigQuery export mode calls get_table() twice. Once to get
+    # metadata (numBytes), and once to retrieve the table's schema
+    # However, the second call is never reached because this test will always
+    # fail before it does so
+    # After the first call, any additional calls are retries
+    self.assertEqual(expected_retries, mock_get_table.call_count - 1)
+
   @parameterized.expand([
       param(
           exception_type=exceptions.BadRequest if exceptions else None,
diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
index 41ef57b5b27c..fab5a8611591 100644
--- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py
+++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py
@@ -751,7 +751,7 @@ def _insert_all_rows(
 
   @retry.with_exponential_backoff(
       num_retries=MAX_RETRIES,
-      retry_filter=retry.retry_on_server_errors_and_timeout_filter)
+      retry_filter=retry.retry_on_server_errors_timeout_or_quota_issues_filter)
   def get_table(self, project_id, dataset_id, table_id):
     """Lookup a table's metadata object.
 
diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py
index 838c956b6cf6..087d32a9e054 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio.py
@@ -40,6 +40,7 @@
 from google.cloud.storage.fileio import BlobReader
 from google.cloud.storage.fileio import BlobWriter
 
+from apache_beam import version as beam_version
 from apache_beam.internal.gcp import auth
 from apache_beam.options.pipeline_options import GoogleCloudOptions
 from apache_beam.options.pipeline_options import PipelineOptions
@@ -111,7 +112,11 @@ def __init__(self, storage_client=None, pipeline_options=None):
       if credentials:
         storage_client = storage.Client(
             credentials=credentials.get_google_auth_credentials(),
-            project=pipeline_options.view_as(GoogleCloudOptions).project)
+            project=pipeline_options.view_as(GoogleCloudOptions).project,
+            extra_headers={
+                "User-Agent": "apache-beam/%s (GPN:Beam)" %
+                beam_version.__version__
+            })
       else:
         storage_client = storage.Client.create_anonymous_client()
     self.client = storage_client
diff --git a/sdks/python/apache_beam/io/gcp/pubsub.py b/sdks/python/apache_beam/io/gcp/pubsub.py
index af58006d6e76..d0785fa1f214 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub.py
@@ -150,7 +150,7 @@ def _to_proto_str(self, for_publish=False):
       containing the payload of this object.
     """
     msg = pubsub.types.PubsubMessage()
-    if len(self.data) > (10 << 20):
+    if len(self.data) > (10_000_000):
       raise ValueError('A pubsub message data field must not exceed 10MB')
     msg.data = self.data
 
@@ -179,7 +179,7 @@ def _to_proto_str(self, for_publish=False):
     msg.ordering_key = self.ordering_key
 
     serialized = pubsub.types.PubsubMessage.serialize(msg)
-    if len(serialized) > (10 << 20):
+    if len(serialized) > (10_000_000):
       raise ValueError(
           'Serialized pubsub message exceeds the publish request limit of 10MB')
     return serialized
diff --git a/sdks/python/apache_beam/io/gcp/spanner.py b/sdks/python/apache_beam/io/gcp/spanner.py
index c16daa4448b1..51c7fc65c171 100644
--- a/sdks/python/apache_beam/io/gcp/spanner.py
+++ b/sdks/python/apache_beam/io/gcp/spanner.py
@@ -124,6 +124,11 @@ class TimestampBoundMode(Enum):
   STRONG = auto()
 
 
+class FailureMode(Enum):
+  FAIL_FAST = auto()
+  REPORT_FAILURES = auto()
+
+
 class ReadFromSpannerSchema(NamedTuple):
   instance_id: str
   database_id: str
@@ -282,6 +287,7 @@ class WriteToSpannerSchema(NamedTuple):
   emulator_host: Optional[str]
   commit_deadline: Optional[int]
   max_cumulative_backoff: Optional[int]
+  failure_mode: Optional[str]
 
 
 _CLASS_DOC = \
@@ -346,6 +352,11 @@ class {row_type}(NamedTuple):
       (15min). If the mutations still have not been written after this time,
       they are treated as a failure, and handled according to the setting of
       failure_mode. Pass seconds as value.
+  :param failure_mode: Specifies the behavior for mutations that fail to be
+      written to Spanner. Default is FAIL_FAST. When FAIL_FAST is set,
+      an exception will be thrown for any failed mutation. When REPORT_FAILURES
+      is set, processing will continue instead of throwing an exception. Note
+      that REPORT_FAILURES can cause data loss if used incorrectly.
   :param expansion_service: The address (host:port) of the ExpansionService.
   """
 
@@ -392,6 +403,7 @@ def __init__(
       emulator_host=None,
       commit_deadline=None,
       max_cumulative_backoff=None,
+      failure_mode=None,
       expansion_service=None,
   ):
     max_cumulative_backoff = int(
@@ -413,6 +425,7 @@ def __init__(
                 emulator_host=emulator_host,
                 commit_deadline=commit_deadline,
                 max_cumulative_backoff=max_cumulative_backoff,
+                failure_mode=_get_enum_name(failure_mode),
             ),
         ),
         expansion_service=expansion_service or default_io_expansion_service(),
@@ -445,6 +458,7 @@ def __init__(
       commit_deadline=None,
       max_cumulative_backoff=None,
       expansion_service=None,
+      failure_mode=None,
   ):
     max_cumulative_backoff = int(
         max_cumulative_backoff) if max_cumulative_backoff else None
@@ -465,6 +479,7 @@ def __init__(
                 emulator_host=emulator_host,
                 commit_deadline=commit_deadline,
                 max_cumulative_backoff=max_cumulative_backoff,
+                failure_mode=_get_enum_name(failure_mode),
             ),
         ),
         expansion_service=expansion_service or default_io_expansion_service(),
@@ -497,6 +512,7 @@ def __init__(
       commit_deadline=None,
       max_cumulative_backoff=None,
       expansion_service=None,
+      failure_mode=None,
   ):
     max_cumulative_backoff = int(
         max_cumulative_backoff) if max_cumulative_backoff else None
@@ -517,6 +533,7 @@ def __init__(
                 emulator_host=emulator_host,
                 commit_deadline=commit_deadline,
                 max_cumulative_backoff=max_cumulative_backoff,
+                failure_mode=_get_enum_name(failure_mode),
             ),
         ),
         expansion_service=expansion_service or default_io_expansion_service(),
@@ -548,6 +565,7 @@ def __init__(
       emulator_host=None,
       commit_deadline=None,
       max_cumulative_backoff=None,
+      failure_mode=None,
       expansion_service=None,
   ):
     max_cumulative_backoff = int(
@@ -569,6 +587,7 @@ def __init__(
                 emulator_host=emulator_host,
                 commit_deadline=commit_deadline,
                 max_cumulative_backoff=max_cumulative_backoff,
+                failure_mode=_get_enum_name(failure_mode),
             ),
         ),
         expansion_service=expansion_service or default_io_expansion_service(),
@@ -600,6 +619,7 @@ def __init__(
       emulator_host=None,
       commit_deadline=None,
       max_cumulative_backoff=None,
+      failure_mode=None,
       expansion_service=None,
   ):
     max_cumulative_backoff = int(
@@ -621,6 +641,7 @@ def __init__(
                 emulator_host=emulator_host,
                 commit_deadline=commit_deadline,
                 max_cumulative_backoff=max_cumulative_backoff,
+                failure_mode=_get_enum_name(failure_mode),
             ),
         ),
         expansion_service=expansion_service or default_io_expansion_service(),
diff --git a/sdks/python/apache_beam/io/gcp/tests/xlang_spannerio_it_test.py b/sdks/python/apache_beam/io/gcp/tests/xlang_spannerio_it_test.py
index 5d701052965b..43a74f170531 100644
--- a/sdks/python/apache_beam/io/gcp/tests/xlang_spannerio_it_test.py
+++ b/sdks/python/apache_beam/io/gcp/tests/xlang_spannerio_it_test.py
@@ -234,6 +234,7 @@ def run_write_pipeline(
               database_id=self.database_id,
               project_id=self.project_id,
               table=self.table,
+              failure_mode=beam.io.gcp.spanner.FailureMode.REPORT_FAILURES,
               emulator_host=self.spanner_helper.get_emulator_host(),
           ))
 
diff --git a/sdks/python/apache_beam/io/parquetio.py b/sdks/python/apache_beam/io/parquetio.py
index 4696e5ae7927..48c51428c17d 100644
--- a/sdks/python/apache_beam/io/parquetio.py
+++ b/sdks/python/apache_beam/io/parquetio.py
@@ -231,9 +231,7 @@ def display_data(self):
 
 
 class ReadFromParquet(PTransform):
-  """A :class:`~apache_beam.transforms.ptransform.PTransform` for reading
-     Parquet files as a `PCollection` of dictionaries. This `PTransform` is
-     currently experimental. No backward-compatibility guarantees."""
+  """A `PTransform` for reading Parquet files."""
   def __init__(
       self,
       file_pattern=None,
@@ -465,9 +463,6 @@ def split_points_unclaimed(stop_position):
 
 class WriteToParquet(PTransform):
   """A ``PTransform`` for writing parquet files.
-
-    This ``PTransform`` is currently experimental. No backward-compatibility
-    guarantees.
   """
   def __init__(
       self,
diff --git a/sdks/python/apache_beam/io/parquetio_test.py b/sdks/python/apache_beam/io/parquetio_test.py
index 1cd5f1208cc2..06d9a5271404 100644
--- a/sdks/python/apache_beam/io/parquetio_test.py
+++ b/sdks/python/apache_beam/io/parquetio_test.py
@@ -334,6 +334,9 @@ def test_write_batched_display_data(self):
     ]
     hc.assert_that(dd.items, hc.contains_inanyorder(*expected_items))
 
+  @unittest.skipIf(
+      ARROW_MAJOR_VERSION >= 13,
+      'pyarrow 13.x and above does not throw ArrowInvalid error')
   def test_sink_transform_int96(self):
     with tempfile.NamedTemporaryFile() as dst:
       path = dst.name
diff --git a/sdks/python/apache_beam/io/requestresponseio.py b/sdks/python/apache_beam/io/requestresponseio.py
index fc742fa00cad..0ec586e64018 100644
--- a/sdks/python/apache_beam/io/requestresponseio.py
+++ b/sdks/python/apache_beam/io/requestresponseio.py
@@ -17,11 +17,24 @@
 
 """``PTransform`` for reading from and writing to Web APIs."""
 import abc
+import concurrent.futures
+import contextlib
+import logging
+import sys
+from typing import Generic
+from typing import Optional
 from typing import TypeVar
 
+import apache_beam as beam
+from apache_beam.pvalue import PCollection
+
 RequestT = TypeVar('RequestT')
 ResponseT = TypeVar('ResponseT')
 
+DEFAULT_TIMEOUT_SECS = 30  # seconds
+
+_LOGGER = logging.getLogger(__name__)
+
 
 class UserCodeExecutionException(Exception):
   """Base class for errors related to calling Web APIs."""
@@ -37,8 +50,10 @@ class UserCodeTimeoutException(UserCodeExecutionException):
   """Extends ``UserCodeExecutionException`` to signal a user code timeout."""
 
 
-class Caller(metaclass=abc.ABCMeta):
-  """Interfaces user custom code intended for API calls."""
+class Caller(contextlib.AbstractContextManager, abc.ABC):
+  """Interface for user custom code intended for API calls.
+  For setup and teardown of clients when applicable, implement the
+  ``__enter__`` and ``__exit__`` methods respectively."""
   @abc.abstractmethod
   def __call__(self, request: RequestT, *args, **kwargs) -> ResponseT:
     """Calls a Web API with the ``RequestT``  and returns a
@@ -48,18 +63,156 @@ def __call__(self, request: RequestT, *args, **kwargs) -> ResponseT:
     """
     pass
 
+  def __enter__(self):
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    return None
+
+
+class ShouldBackOff(abc.ABC):
+  """
+  ShouldBackOff provides mechanism to apply adaptive throttling.
+  """
+  pass
+
+
+class Repeater(abc.ABC):
+  """Repeater provides mechanism to repeat requests for a
+  configurable condition."""
+  pass
 
-class SetupTeardown(metaclass=abc.ABCMeta):
-  """Interfaces user custom code to set up and teardown the API clients.
-    Called by ``RequestResponseIO`` within its DoFn's setup and teardown
-    methods.
+
+class CacheReader(abc.ABC):
+  """CacheReader provides mechanism to read from the cache."""
+  pass
+
+
+class CacheWriter(abc.ABC):
+  """CacheWriter provides mechanism to write to the cache."""
+  pass
+
+
+class PreCallThrottler(abc.ABC):
+  """PreCallThrottler provides a throttle mechanism before sending request."""
+  pass
+
+
+class RequestResponseIO(beam.PTransform[beam.PCollection[RequestT],
+                                        beam.PCollection[ResponseT]]):
+  """A :class:`RequestResponseIO` transform to read and write to APIs.
+
+  Processes an input :class:`~apache_beam.pvalue.PCollection` of requests
+  by making a call to the API as defined in :class:`Caller`'s `__call__`
+  and returns a :class:`~apache_beam.pvalue.PCollection` of responses.
+  """
+  def __init__(
+      self,
+      caller: [Caller],
+      timeout: Optional[float] = DEFAULT_TIMEOUT_SECS,
+      should_backoff: Optional[ShouldBackOff] = None,
+      repeater: Optional[Repeater] = None,
+      cache_reader: Optional[CacheReader] = None,
+      cache_writer: Optional[CacheWriter] = None,
+      throttler: Optional[PreCallThrottler] = None,
+  ):
     """
-  @abc.abstractmethod
-  def setup(self) -> None:
-    """Called during the DoFn's setup lifecycle method."""
-    pass
+    Instantiates a RequestResponseIO transform.
 
-  @abc.abstractmethod
-  def teardown(self) -> None:
-    """Called during the DoFn's teardown lifecycle method."""
-    pass
+    Args:
+      caller (~apache_beam.io.requestresponseio.Caller): an implementation of
+        `Caller` object that makes call to the API.
+      timeout (float): timeout value in seconds to wait for response from API.
+      should_backoff (~apache_beam.io.requestresponseio.ShouldBackOff):
+        (Optional) provides methods for backoff.
+      repeater (~apache_beam.io.requestresponseio.Repeater): (Optional)
+        provides methods to repeat requests to API.
+      cache_reader (~apache_beam.io.requestresponseio.CacheReader): (Optional)
+        provides methods to read external cache.
+      cache_writer (~apache_beam.io.requestresponseio.CacheWriter): (Optional)
+        provides methods to write to external cache.
+      throttler (~apache_beam.io.requestresponseio.PreCallThrottler):
+        (Optional) provides methods to pre-throttle a request.
+    """
+    self._caller = caller
+    self._timeout = timeout
+    self._should_backoff = should_backoff
+    self._repeater = repeater
+    self._cache_reader = cache_reader
+    self._cache_writer = cache_writer
+    self._throttler = throttler
+
+  def expand(self, requests: PCollection[RequestT]) -> PCollection[ResponseT]:
+    # TODO(riteshghorse): add Cache and Throttle PTransforms.
+    return requests | _Call(
+        caller=self._caller,
+        timeout=self._timeout,
+        should_backoff=self._should_backoff,
+        repeater=self._repeater)
+
+
+class _Call(beam.PTransform[beam.PCollection[RequestT],
+                            beam.PCollection[ResponseT]]):
+  """(Internal-only) PTransform that invokes a remote function on each element
+   of the input PCollection.
+
+  This PTransform uses a `Caller` object to invoke the actual API calls,
+  and uses ``__enter__`` and ``__exit__`` to manage setup and teardown of
+  clients when applicable. Additionally, a timeout value is specified to
+  regulate the duration of each call, defaults to 30 seconds.
+
+  Args:
+      caller (:class:`apache_beam.io.requestresponseio.Caller`): a callable
+        object that invokes API call.
+      timeout (float): timeout value in seconds to wait for response from API.
+  """
+  def __init__(
+      self,
+      caller: Caller,
+      timeout: Optional[float] = DEFAULT_TIMEOUT_SECS,
+      should_backoff: Optional[ShouldBackOff] = None,
+      repeater: Optional[Repeater] = None,
+  ):
+    """Initialize the _Call transform.
+    Args:
+      caller (:class:`apache_beam.io.requestresponseio.Caller`): a callable
+        object that invokes API call.
+      timeout (float): timeout value in seconds to wait for response from API.
+      should_backoff (~apache_beam.io.requestresponseio.ShouldBackOff):
+        (Optional) provides methods for backoff.
+      repeater (~apache_beam.io.requestresponseio.Repeater): (Optional) provides
+        methods to repeat requests to API.
+    """
+    self._caller = caller
+    self._timeout = timeout
+    self._should_backoff = should_backoff
+    self._repeater = repeater
+
+  def expand(
+      self,
+      requests: beam.PCollection[RequestT]) -> beam.PCollection[ResponseT]:
+    return requests | beam.ParDo(_CallDoFn(self._caller, self._timeout))
+
+
+class _CallDoFn(beam.DoFn, Generic[RequestT, ResponseT]):
+  def setup(self):
+    self._caller.__enter__()
+
+  def __init__(self, caller: Caller, timeout: float):
+    self._caller = caller
+    self._timeout = timeout
+
+  def process(self, request, *args, **kwargs):
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+      future = executor.submit(self._caller, request)
+      try:
+        yield future.result(timeout=self._timeout)
+      except concurrent.futures.TimeoutError:
+        raise UserCodeTimeoutException(
+            f'Timeout {self._timeout} exceeded '
+            f'while completing request: {request}')
+      except RuntimeError:
+        raise UserCodeExecutionException('could not complete request')
+
+  def teardown(self):
+    self._caller.__exit__(*sys.exc_info())
diff --git a/sdks/python/apache_beam/io/requestresponseio_it_test.py b/sdks/python/apache_beam/io/requestresponseio_it_test.py
index f291ff96a4d7..aae6b4e6ef2c 100644
--- a/sdks/python/apache_beam/io/requestresponseio_it_test.py
+++ b/sdks/python/apache_beam/io/requestresponseio_it_test.py
@@ -23,10 +23,13 @@
 
 import urllib3
 
+import apache_beam as beam
 from apache_beam.io.requestresponseio import Caller
+from apache_beam.io.requestresponseio import RequestResponseIO
 from apache_beam.io.requestresponseio import UserCodeExecutionException
 from apache_beam.io.requestresponseio import UserCodeQuotaException
 from apache_beam.options.pipeline_options import PipelineOptions
+from apache_beam.testing.test_pipeline import TestPipeline
 
 _HTTP_PATH = '/v1/echo'
 _PAYLOAD = base64.b64encode(bytes('payload', 'utf-8'))
@@ -86,7 +89,6 @@ def __call__(self, request: EchoRequest, *args, **kwargs) -> EchoResponse:
         ``UserCodeExecutionException``, ``UserCodeTimeoutException``,
         or a ``UserCodeQuotaException``.
         """
-
     try:
       resp = urllib3.request(
           "POST",
@@ -104,8 +106,8 @@ def __call__(self, request: EchoRequest, *args, **kwargs) -> EchoResponse:
 
       if resp.status == 429:  # Too Many Requests
         raise UserCodeQuotaException(resp.reason)
-
-      raise UserCodeExecutionException(resp.reason)
+      else:
+        raise UserCodeExecutionException(resp.status, resp.reason, request)
 
     except urllib3.exceptions.HTTPError as e:
       raise UserCodeExecutionException(e)
@@ -167,6 +169,16 @@ def test_not_found_should_raise(self):
     self.assertRaisesRegex(
         UserCodeExecutionException, "Not Found", lambda: client(req))
 
+  def test_request_response_io(self):
+    client, options = EchoHTTPCallerTestIT._get_client_and_options()
+    req = EchoRequest(id=options.never_exceed_quota_id, payload=_PAYLOAD)
+    with TestPipeline(is_integration_test=True) as test_pipeline:
+      output = (
+          test_pipeline
+          | 'Create PCollection' >> beam.Create([req])
+          | 'RRIO Transform' >> RequestResponseIO(client))
+      self.assertIsNotNone(output)
+
 
 if __name__ == '__main__':
   unittest.main(argv=sys.argv[:1])
diff --git a/sdks/python/apache_beam/io/requestresponseio_test.py b/sdks/python/apache_beam/io/requestresponseio_test.py
new file mode 100644
index 000000000000..2828a3578871
--- /dev/null
+++ b/sdks/python/apache_beam/io/requestresponseio_test.py
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import time
+import unittest
+
+import apache_beam as beam
+from apache_beam.io.requestresponseio import Caller
+from apache_beam.io.requestresponseio import RequestResponseIO
+from apache_beam.io.requestresponseio import UserCodeExecutionException
+from apache_beam.io.requestresponseio import UserCodeTimeoutException
+from apache_beam.testing.test_pipeline import TestPipeline
+
+
+class AckCaller(Caller):
+  """AckCaller acknowledges the incoming request by returning a
+  request with ACK."""
+  def __enter__(self):
+    pass
+
+  def __call__(self, request: str):
+    return f"ACK: {request}"
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    return None
+
+
+class CallerWithTimeout(AckCaller):
+  """CallerWithTimeout sleeps for 2 seconds before responding.
+  Used to test timeout in RequestResponseIO."""
+  def __call__(self, request: str, *args, **kwargs):
+    time.sleep(2)
+    return f"ACK: {request}"
+
+
+class CallerWithRuntimeError(AckCaller):
+  """CallerWithRuntimeError raises a `RuntimeError` for RequestResponseIO
+  to raise a UserCodeExecutionException."""
+  def __call__(self, request: str, *args, **kwargs):
+    if not request:
+      raise RuntimeError("Exception expected, not an error.")
+
+
+class TestCaller(unittest.TestCase):
+  def test_valid_call(self):
+    caller = AckCaller()
+    with TestPipeline() as test_pipeline:
+      output = (
+          test_pipeline
+          | beam.Create(["sample_request"])
+          | RequestResponseIO(caller=caller))
+
+    self.assertIsNotNone(output)
+
+  def test_call_timeout(self):
+    caller = CallerWithTimeout()
+    with self.assertRaises(UserCodeTimeoutException):
+      with TestPipeline() as test_pipeline:
+        _ = (
+            test_pipeline
+            | beam.Create(["timeout_request"])
+            | RequestResponseIO(caller=caller, timeout=1))
+
+  def test_call_runtime_error(self):
+    caller = CallerWithRuntimeError()
+    with self.assertRaises(UserCodeExecutionException):
+      with TestPipeline() as test_pipeline:
+        _ = (
+            test_pipeline
+            | beam.Create([""])
+            | RequestResponseIO(caller=caller))
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/sdks/python/apache_beam/ml/inference/__init__.py b/sdks/python/apache_beam/ml/inference/__init__.py
index 0433761ded8a..3ba45ad7def0 100644
--- a/sdks/python/apache_beam/ml/inference/__init__.py
+++ b/sdks/python/apache_beam/ml/inference/__init__.py
@@ -21,7 +21,7 @@
 
 Note: on top of the frameworks captured in submodules below, Beam also has
 a supported TensorFlow model handler via the tfx-bsl library. See
-https://beam.apache.org/documentation/sdks/python-machine-learning/#tensorflow
+https://beam.apache.org/documentation/ml/about-ml/#tensorflow
 for more information on using TensorFlow in Beam.
 """
 
diff --git a/sdks/python/apache_beam/ml/inference/base.py b/sdks/python/apache_beam/ml/inference/base.py
index 90ba22fcd314..63038f9c9b7d 100644
--- a/sdks/python/apache_beam/ml/inference/base.py
+++ b/sdks/python/apache_beam/ml/inference/base.py
@@ -143,7 +143,7 @@ class KeyModelPathMapping(Generic[KeyT]):
   information see the KeyedModelHandler documentation
   https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.KeyedModelHandler
   documentation and the website section on model updates
-  https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh
+  https://beam.apache.org/documentation/ml/about-ml/#automatic-model-refresh
   """
   keys: List[KeyT]
   update_path: str
@@ -224,7 +224,7 @@ def update_model_path(self, model_path: Optional[str] = None):
     used when a ModelHandler represents a single model, not multiple models.
     This will be true in most cases. For more information see the website
     section on model updates
-    https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh
+    https://beam.apache.org/documentation/ml/about-ml/#automatic-model-refresh
     """
     pass
 
@@ -239,7 +239,7 @@ def update_model_paths(
     the KeyedModelHandler documentation
     https://beam.apache.org/releases/pydoc/current/apache_beam.ml.inference.base.html#apache_beam.ml.inference.base.KeyedModelHandler
     documentation and the website section on model updates
-    https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh
+    https://beam.apache.org/documentation/ml/about-ml/#automatic-model-refresh
     """
     pass
 
@@ -452,7 +452,7 @@ def __init__(
     KeyedModelHandlers support Automatic Model Refresh to update your model
     to a newer version without stopping your streaming pipeline. For an
     overview of this feature, see
-    https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh
+    https://beam.apache.org/documentation/ml/about-ml/#automatic-model-refresh
 
 
     To use this feature with a KeyedModelHandler that has many models per key,
@@ -465,7 +465,7 @@ def __init__(
     will update the model corresponding to keys 'k1' and 'k2' with path
     'update/path/1' and the model corresponding to 'k3' with 'update/path/2'.
     In order to do a side input update: (1) all restrictions mentioned in
-    https://beam.apache.org/documentation/sdks/python-machine-learning/#automatic-model-refresh
+    https://beam.apache.org/documentation/ml/about-ml/#automatic-model-refresh
     must be met, (2) all update_paths must be non-empty, even if they are not
     being updated from their original values, and (3) the set of keys
     originally defined cannot change. This means that if originally you have
@@ -486,7 +486,7 @@ def __init__(
     memory (OOM) exception. To avoid this issue, use the parameter
     `max_models_per_worker_hint` to limit the number of models that are loaded
     at the same time. For more information about memory management, see
-    `Use a keyed `ModelHandler <https://beam.apache.org/documentation/sdks/python-machine-learning/#use-a-keyed-modelhandler>_`.  # pylint: disable=line-too-long
+    `Use a keyed `ModelHandler <https://beam.apache.org/documentation/ml/about-ml/#use-a-keyed-modelhandler-object>_`.  # pylint: disable=line-too-long
 
 
     Args:
@@ -498,7 +498,7 @@ def __init__(
         example, if your worker has 8 GB of memory provisioned and your workers
         take up 1 GB each, you should set this to 7 to allow all models to sit
         in memory with some buffer. For more information about memory management,
-        see `Use a keyed `ModelHandler <https://beam.apache.org/documentation/sdks/python-machine-learning/#use-a-keyed-modelhandler>_`.  # pylint: disable=line-too-long
+        see `Use a keyed `ModelHandler <https://beam.apache.org/documentation/ml/about-ml/#use-a-keyed-modelhandler-object>_`.  # pylint: disable=line-too-long
     """
     self._metrics_collectors: Dict[str, _MetricsCollector] = {}
     self._default_metrics_collector: _MetricsCollector = None
diff --git a/sdks/python/apache_beam/ml/inference/huggingface_inference.py b/sdks/python/apache_beam/ml/inference/huggingface_inference.py
index 878d7bfc9cf2..25367d22eaa3 100644
--- a/sdks/python/apache_beam/ml/inference/huggingface_inference.py
+++ b/sdks/python/apache_beam/ml/inference/huggingface_inference.py
@@ -221,9 +221,9 @@ def __init__(
       *,
       inference_fn: Optional[Callable[..., Iterable[PredictionResult]]] = None,
       load_model_args: Optional[Dict[str, Any]] = None,
-      inference_args: Optional[Dict[str, Any]] = None,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """
@@ -249,12 +249,10 @@ def __init__(
       load_model_args (Dict[str, Any]): (Optional) Keyword arguments to provide
         load options while loading models from Hugging Face Hub.
         Defaults to None.
-      inference_args (Dict[str, Any]): (Optional) Non-batchable arguments
-        required as inputs to the model's inference function. Unlike Tensors
-        in `batch`, these parameters will not be dynamically batched.
-        Defaults to None.
       min_batch_size: the minimum batch size to use when batching inputs.
       max_batch_size: the maximum batch size to use when batching inputs.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+        before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -270,13 +268,14 @@ def __init__(
     self._device = device
     self._inference_fn = inference_fn
     self._model_config_args = load_model_args if load_model_args else {}
-    self._inference_args = inference_args if inference_args else {}
     self._batching_kwargs = {}
     self._env_vars = kwargs.get("env_vars", {})
     if min_batch_size is not None:
       self._batching_kwargs["min_batch_size"] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs["max_batch_size"] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._large_model = large_model
     self._framework = framework
 
@@ -288,7 +287,7 @@ def load_model(self):
     model = self._model_class.from_pretrained(
         self._model_uri, **self._model_config_args)
     if self._framework == 'pt':
-      if self._device == "GPU" and is_gpu_available_torch:
+      if self._device == "GPU" and is_gpu_available_torch():
         model.to(torch.device("cuda"))
       if callable(getattr(model, 'requires_grad_', None)):
         model.requires_grad_(False)
@@ -402,9 +401,9 @@ def __init__(
       *,
       inference_fn: Optional[Callable[..., Iterable[PredictionResult]]] = None,
       load_model_args: Optional[Dict[str, Any]] = None,
-      inference_args: Optional[Dict[str, Any]] = None,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """
@@ -430,12 +429,10 @@ def __init__(
       load_model_args (Dict[str, Any]): (Optional) keyword arguments to provide
         load options while loading models from Hugging Face Hub.
         Defaults to None.
-      inference_args (Dict[str, Any]): (Optional) Non-batchable arguments
-        required as inputs to the model's inference function. Unlike Tensors
-        in `batch`, these parameters will not be dynamically batched.
-        Defaults to None.
       min_batch_size: the minimum batch size to use when batching inputs.
       max_batch_size: the maximum batch size to use when batching inputs.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+        before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -451,13 +448,14 @@ def __init__(
     self._device = device
     self._inference_fn = inference_fn
     self._model_config_args = load_model_args if load_model_args else {}
-    self._inference_args = inference_args if inference_args else {}
     self._batching_kwargs = {}
     self._env_vars = kwargs.get("env_vars", {})
     if min_batch_size is not None:
       self._batching_kwargs["min_batch_size"] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs["max_batch_size"] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._large_model = large_model
     self._framework = ""
 
@@ -576,9 +574,9 @@ def __init__(
       device: Optional[str] = None,
       inference_fn: PipelineInferenceFn = _default_pipeline_inference_fn,
       load_pipeline_args: Optional[Dict[str, Any]] = None,
-      inference_args: Optional[Dict[str, Any]] = None,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """
@@ -612,11 +610,10 @@ def __init__(
         Default is _default_pipeline_inference_fn.
       load_pipeline_args (Dict[str, Any]): keyword arguments to provide load
         options while loading pipelines from Hugging Face. Defaults to None.
-      inference_args (Dict[str, Any]): Non-batchable arguments
-        required as inputs to the model's inference function.
-        Defaults to None.
       min_batch_size: the minimum batch size to use when batching inputs.
       max_batch_size: the maximum batch size to use when batching inputs.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+        before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -631,7 +628,6 @@ def __init__(
     self._model = model
     self._inference_fn = inference_fn
     self._load_pipeline_args = load_pipeline_args if load_pipeline_args else {}
-    self._inference_args = inference_args if inference_args else {}
     self._batching_kwargs = {}
     self._framework = "torch"
     self._env_vars = kwargs.get('env_vars', {})
@@ -639,6 +635,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._large_model = large_model
 
     # Check if the device is specified twice. If true then the device parameter
diff --git a/sdks/python/apache_beam/ml/inference/onnx_inference.py b/sdks/python/apache_beam/ml/inference/onnx_inference.py
index 18e115a61887..f7b6c0115af8 100644
--- a/sdks/python/apache_beam/ml/inference/onnx_inference.py
+++ b/sdks/python/apache_beam/ml/inference/onnx_inference.py
@@ -19,6 +19,7 @@
 from typing import Callable
 from typing import Dict
 from typing import Iterable
+from typing import Mapping
 from typing import Optional
 from typing import Sequence
 
@@ -63,6 +64,9 @@ def __init__( #pylint: disable=dangerous-default-value
       *,
       inference_fn: NumpyInferenceFn = default_numpy_inference_fn,
       large_model: bool = False,
+      min_batch_size: Optional[int] = None,
+      max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       **kwargs):
     """ Implementation of the ModelHandler interface for onnx
     using numpy arrays as input.
@@ -80,6 +84,10 @@ def __init__( #pylint: disable=dangerous-default-value
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
         set this to True if N*W > M.
+      min_batch_size: the minimum batch size to use when batching inputs.
+      max_batch_size: the maximum batch size to use when batching inputs.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+        before emitting; used in streaming contexts.
       kwargs: 'env_vars' can be used to set environment variables
         before loading the model.
     """
@@ -90,6 +98,13 @@ def __init__( #pylint: disable=dangerous-default-value
     self._model_inference_fn = inference_fn
     self._env_vars = kwargs.get('env_vars', {})
     self._large_model = large_model
+    self._batching_kwargs = {}
+    if min_batch_size is not None:
+      self._batching_kwargs["min_batch_size"] = min_batch_size
+    if max_batch_size is not None:
+      self._batching_kwargs["max_batch_size"] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
 
   def load_model(self) -> ort.InferenceSession:
     """Loads and initializes an onnx inference session for processing."""
@@ -143,3 +158,6 @@ def get_metrics_namespace(self) -> str:
 
   def share_model_across_processes(self) -> bool:
     return self._large_model
+
+  def batch_elements_kwargs(self) -> Mapping[str, Any]:
+    return self._batching_kwargs
diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference.py b/sdks/python/apache_beam/ml/inference/pytorch_inference.py
index 26e593fdd7de..480dc538195c 100644
--- a/sdks/python/apache_beam/ml/inference/pytorch_inference.py
+++ b/sdks/python/apache_beam/ml/inference/pytorch_inference.py
@@ -193,6 +193,7 @@ def __init__(
       torch_script_model_path: Optional[str] = None,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       load_model_args: Optional[Dict[str, Any]] = None,
       **kwargs):
@@ -227,6 +228,8 @@ def __init__(
         batch will be fed into the inference_fn as a Sequence of Tensors.
       max_batch_size: the maximum batch size to use when batching inputs. This
         batch will be fed into the inference_fn as a Sequence of Tensors.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+          before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -254,6 +257,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._torch_script_model_path = torch_script_model_path
     self._load_model_args = load_model_args if load_model_args else {}
     self._env_vars = kwargs.get('env_vars', {})
@@ -421,6 +426,7 @@ def __init__(
       torch_script_model_path: Optional[str] = None,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       load_model_args: Optional[Dict[str, Any]] = None,
       **kwargs):
@@ -460,6 +466,8 @@ def __init__(
         batch will be fed into the inference_fn as a Sequence of Keyed Tensors.
       max_batch_size: the maximum batch size to use when batching inputs. This
         batch will be fed into the inference_fn as a Sequence of Keyed Tensors.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+          before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -487,6 +495,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._torch_script_model_path = torch_script_model_path
     self._load_model_args = load_model_args if load_model_args else {}
     self._env_vars = kwargs.get('env_vars', {})
diff --git a/sdks/python/apache_beam/ml/inference/sklearn_inference.py b/sdks/python/apache_beam/ml/inference/sklearn_inference.py
index c2bd2cee66e6..befeca7f33b2 100644
--- a/sdks/python/apache_beam/ml/inference/sklearn_inference.py
+++ b/sdks/python/apache_beam/ml/inference/sklearn_inference.py
@@ -90,6 +90,7 @@ def __init__(
       inference_fn: NumpyInferenceFn = _default_numpy_inference_fn,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """ Implementation of the ModelHandler interface for scikit-learn
@@ -111,6 +112,8 @@ def __init__(
       max_batch_size: the maximum batch size to use when batching inputs. This
         batch will be fed into the inference_fn as a Sequence of Numpy
         ndarrays.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+          before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -126,6 +129,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._env_vars = kwargs.get('env_vars', {})
     self._large_model = large_model
 
@@ -212,6 +217,7 @@ def __init__(
       inference_fn: PandasInferenceFn = _default_pandas_inference_fn,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """Implementation of the ModelHandler interface for scikit-learn that
@@ -236,6 +242,8 @@ def __init__(
       max_batch_size: the maximum batch size to use when batching inputs. This
         batch will be fed into the inference_fn as a Sequence of Pandas
         Dataframes.
+      max_batch_duration_secs: the maximum amount of time to buffer a batch
+          before emitting; used in streaming contexts.
       large_model: set to true if your model is large enough to run into
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
@@ -251,6 +259,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._env_vars = kwargs.get('env_vars', {})
     self._large_model = large_model
 
diff --git a/sdks/python/apache_beam/ml/inference/tensorflow_inference.py b/sdks/python/apache_beam/ml/inference/tensorflow_inference.py
index 991ae971d9e6..0802868a1dd4 100644
--- a/sdks/python/apache_beam/ml/inference/tensorflow_inference.py
+++ b/sdks/python/apache_beam/ml/inference/tensorflow_inference.py
@@ -110,6 +110,7 @@ def __init__(
       inference_fn: TensorInferenceFn = default_numpy_inference_fn,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """Implementation of the ModelHandler interface for Tensorflow.
@@ -154,6 +155,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._large_model = large_model
 
   def load_model(self) -> tf.Module:
@@ -235,6 +238,7 @@ def __init__(
       inference_fn: TensorInferenceFn = default_tensor_inference_fn,
       min_batch_size: Optional[int] = None,
       max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       large_model: bool = False,
       **kwargs):
     """Implementation of the ModelHandler interface for Tensorflow.
@@ -258,6 +262,10 @@ def __init__(
           once the model is loaded.
         inference_fn: inference function to use during RunInference.
           Defaults to default_numpy_inference_fn.
+        min_batch_size: the minimum batch size to use when batching inputs.
+        max_batch_size: the maximum batch size to use when batching inputs.
+        max_batch_duration_secs: the maximum amount of time to buffer a batch
+          before emitting; used in streaming contexts.
         large_model: set to true if your model is large enough to run into
           memory pressure if you load multiple copies. Given a model that
           consumes N memory and a machine with W cores and M memory, you should
@@ -280,6 +288,8 @@ def __init__(
       self._batching_kwargs['min_batch_size'] = min_batch_size
     if max_batch_size is not None:
       self._batching_kwargs['max_batch_size'] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
     self._large_model = large_model
 
   def load_model(self) -> tf.Module:
diff --git a/sdks/python/apache_beam/ml/inference/tensorrt_inference.py b/sdks/python/apache_beam/ml/inference/tensorrt_inference.py
index ff9bb78d5795..53b81c0c36c1 100644
--- a/sdks/python/apache_beam/ml/inference/tensorrt_inference.py
+++ b/sdks/python/apache_beam/ml/inference/tensorrt_inference.py
@@ -230,6 +230,7 @@ def __init__(
       *,
       inference_fn: TensorRTInferenceFn = _default_tensorRT_inference_fn,
       large_model: bool = False,
+      max_batch_duration_secs: Optional[int] = None,
       **kwargs):
     """Implementation of the ModelHandler interface for TensorRT.
 
@@ -253,6 +254,8 @@ def __init__(
         memory pressure if you load multiple copies. Given a model that
         consumes N memory and a machine with W cores and M memory, you should
         set this to True if N*W > M.
+      max_batch_duration_secs: the maximum amount of time to buffer 
+        a batch before emitting; used in streaming contexts.
       kwargs: Additional arguments like 'engine_path' and 'onnx_path' are
         currently supported. 'env_vars' can be used to set environment variables
         before loading the model.
@@ -262,6 +265,7 @@ def __init__(
     """
     self.min_batch_size = min_batch_size
     self.max_batch_size = max_batch_size
+    self.max_batch_duration_secs = max_batch_duration_secs
     self.inference_fn = inference_fn
     if 'engine_path' in kwargs:
       self.engine_path = kwargs.get('engine_path')
@@ -274,7 +278,8 @@ def batch_elements_kwargs(self):
     """Sets min_batch_size and max_batch_size of a TensorRT engine."""
     return {
         'min_batch_size': self.min_batch_size,
-        'max_batch_size': self.max_batch_size
+        'max_batch_size': self.max_batch_size,
+        'max_batch_duration_secs': self.max_batch_duration_secs
     }
 
   def load_model(self) -> TensorRTEngine:
diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
index 8c902421f603..95660441a848 100644
--- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
+++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference.py
@@ -20,6 +20,7 @@
 from typing import Any
 from typing import Dict
 from typing import Iterable
+from typing import Mapping
 from typing import Optional
 from typing import Sequence
 
@@ -69,6 +70,10 @@ def __init__(
       experiment: Optional[str] = None,
       network: Optional[str] = None,
       private: bool = False,
+      *,
+      min_batch_size: Optional[int] = None,
+      max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       **kwargs):
     """Implementation of the ModelHandler interface for Vertex AI.
     **NOTE:** This API and its implementation are under development and
@@ -97,9 +102,21 @@ def __init__(
       private: optional. if the deployed Vertex AI endpoint is
         private, set to true. Requires a network to be provided
         as well.
+      min_batch_size: optional. the minimum batch size to use when batching
+        inputs.
+      max_batch_size: optional. the maximum batch size to use when batching
+        inputs.
+      max_batch_duration_secs: optional. the maximum amount of time to buffer 
+        a batch before emitting; used in streaming contexts.
     """
-
+    self._batching_kwargs = {}
     self._env_vars = kwargs.get('env_vars', {})
+    if min_batch_size is not None:
+      self._batching_kwargs["min_batch_size"] = min_batch_size
+    if max_batch_size is not None:
+      self._batching_kwargs["max_batch_size"] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
 
     if private and network is None:
       raise ValueError(
@@ -231,3 +248,6 @@ def run_inference(
 
   def validate_inference_args(self, inference_args: Optional[Dict[str, Any]]):
     pass
+
+  def batch_elements_kwargs(self) -> Mapping[str, Any]:
+    return self._batching_kwargs
diff --git a/sdks/python/apache_beam/ml/inference/xgboost_inference.py b/sdks/python/apache_beam/ml/inference/xgboost_inference.py
index 374980c56ece..ff6f098b4150 100644
--- a/sdks/python/apache_beam/ml/inference/xgboost_inference.py
+++ b/sdks/python/apache_beam/ml/inference/xgboost_inference.py
@@ -21,6 +21,7 @@
 from typing import Callable
 from typing import Dict
 from typing import Iterable
+from typing import Mapping
 from typing import Optional
 from typing import Sequence
 from typing import Union
@@ -75,6 +76,10 @@ def __init__(
                          Callable[..., xgboost.XGBModel]],
       model_state: str,
       inference_fn: XGBoostInferenceFn = default_xgboost_inference_fn,
+      *,
+      min_batch_size: Optional[int] = None,
+      max_batch_size: Optional[int] = None,
+      max_batch_duration_secs: Optional[int] = None,
       **kwargs):
     """Implementation of the ModelHandler interface for XGBoost.
 
@@ -95,6 +100,12 @@ def __init__(
         configuration.
       inference_fn: the inference function to use during RunInference.
         default=default_xgboost_inference_fn
+      min_batch_size: optional. the minimum batch size to use when batching
+        inputs.
+      max_batch_size: optional. the maximum batch size to use when batching
+        inputs.
+      max_batch_duration_secs: optional. the maximum amount of time to buffer 
+        a batch before emitting; used in streaming contexts.
       kwargs: 'env_vars' can be used to set environment variables
         before loading the model.
 
@@ -115,6 +126,13 @@ def __init__(
     self._model_state = model_state
     self._inference_fn = inference_fn
     self._env_vars = kwargs.get('env_vars', {})
+    self._batching_kwargs = {}
+    if min_batch_size is not None:
+      self._batching_kwargs["min_batch_size"] = min_batch_size
+    if max_batch_size is not None:
+      self._batching_kwargs["max_batch_size"] = max_batch_size
+    if max_batch_duration_secs is not None:
+      self._batching_kwargs["max_batch_duration_secs"] = max_batch_duration_secs
 
   def load_model(self) -> Union[xgboost.Booster, xgboost.XGBModel]:
     model = self._model_class()
@@ -129,6 +147,9 @@ def load_model(self) -> Union[xgboost.Booster, xgboost.XGBModel]:
   def get_metrics_namespace(self) -> str:
     return 'BeamML_XGBoost'
 
+  def batch_elements_kwargs(self) -> Mapping[str, Any]:
+    return self._batching_kwargs
+
 
 class XGBoostModelHandlerNumpy(XGBoostModelHandler[numpy.ndarray,
                                                    PredictionResult,
diff --git a/sdks/python/apache_beam/ml/transforms/base.py b/sdks/python/apache_beam/ml/transforms/base.py
index b3a30bb5f125..e436ef8cba69 100644
--- a/sdks/python/apache_beam/ml/transforms/base.py
+++ b/sdks/python/apache_beam/ml/transforms/base.py
@@ -14,20 +14,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# pytype: skip-file
-
 import abc
+import collections
+import logging
+import os
+import tempfile
+import uuid
+from typing import Any
 from typing import Dict
 from typing import Generic
 from typing import List
+from typing import Mapping
 from typing import Optional
 from typing import Sequence
 from typing import TypeVar
+from typing import Union
+
+import jsonpickle
+import numpy as np
 
 import apache_beam as beam
+from apache_beam.io.filesystems import FileSystems
 from apache_beam.metrics.metric import Metrics
+from apache_beam.ml.inference.base import ModelHandler
+from apache_beam.ml.inference.base import ModelT
+from apache_beam.options.pipeline_options import PipelineOptions
 
-__all__ = ['MLTransform', 'ProcessHandler', 'BaseOperation']
+_LOGGER = logging.getLogger(__name__)
+_ATTRIBUTE_FILE_NAME = 'attributes.json'
+
+__all__ = [
+    'MLTransform',
+    'ProcessHandler',
+    'MLTransformProvider',
+    'BaseOperation',
+    'EmbeddingsManager'
+]
 
 TransformedDatasetT = TypeVar('TransformedDatasetT')
 TransformedMetadataT = TypeVar('TransformedMetadataT')
@@ -42,12 +64,62 @@
 OperationOutputT = TypeVar('OperationOutputT')
 
 
+def _convert_list_of_dicts_to_dict_of_lists(
+    list_of_dicts: Sequence[Dict[str, Any]]) -> Dict[str, List[Any]]:
+  keys_to_element_list = collections.defaultdict(list)
+  for d in list_of_dicts:
+    for key, value in d.items():
+      keys_to_element_list[key].append(value)
+  return keys_to_element_list
+
+
+def _convert_dict_of_lists_to_lists_of_dict(
+    dict_of_lists: Dict[str, List[Any]]) -> List[Dict[str, Any]]:
+  batch_length = len(next(iter(dict_of_lists.values())))
+  result: List[Dict[str, Any]] = [{} for _ in range(batch_length)]
+  # all the values in the dict_of_lists should have same length
+  for key, values in dict_of_lists.items():
+    assert len(values) == batch_length, (
+        "This function expects all the values "
+        "in the dict_of_lists to have same length."
+        )
+    for i in range(len(values)):
+      result[i][key] = values[i]
+  return result
+
+
 class ArtifactMode(object):
   PRODUCE = 'produce'
   CONSUME = 'consume'
 
 
-class BaseOperation(Generic[OperationInputT, OperationOutputT], abc.ABC):
+class MLTransformProvider:
+  """
+  Data processing transforms that are intended to be used with MLTransform
+  should subclass MLTransformProvider and implement
+  get_ptransform_for_processing().
+
+  get_ptransform_for_processing() method should return a PTransform that can be
+  used to process the data.
+
+  """
+  @abc.abstractmethod
+  def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
+    """
+    Returns a PTransform that can be used to process the data.
+    """
+
+  def get_counter(self):
+    """
+    Returns the counter name for the data processing transform.
+    """
+    counter_name = self.__class__.__name__
+    return Metrics.counter(MLTransform, f'BeamML_{counter_name}')
+
+
+class BaseOperation(Generic[OperationInputT, OperationOutputT],
+                    MLTransformProvider,
+                    abc.ABC):
   def __init__(self, columns: List[str]) -> None:
     """
     Base Opertation class data processing transformations.
@@ -76,33 +148,53 @@ def __call__(self, data: OperationInputT,
     transformed_data = self.apply_transform(data, output_column_name)
     return transformed_data
 
-  def get_counter(self):
-    """
-    Returns the counter name for the operation.
-    """
-    counter_name = self.__class__.__name__
-    return Metrics.counter(MLTransform, f'BeamML_{counter_name}')
-
 
-class ProcessHandler(Generic[ExampleT, MLTransformOutputT], abc.ABC):
+class ProcessHandler(beam.PTransform[beam.PCollection[ExampleT],
+                                     beam.PCollection[MLTransformOutputT]],
+                     abc.ABC):
   """
   Only for internal use. No backwards compatibility guarantees.
   """
   @abc.abstractmethod
-  def process_data(
-      self, pcoll: beam.PCollection[ExampleT]
-  ) -> beam.PCollection[MLTransformOutputT]:
+  def append_transform(self, transform: BaseOperation):
     """
-    Logic to process the data. This will be the entrypoint in
-    beam.MLTransform to process incoming data.
+    Append transforms to the ProcessHandler.
     """
 
+
+# TODO:https://github.com/apache/beam/issues/29356
+#  Add support for inference_fn
+class EmbeddingsManager(MLTransformProvider):
+  def __init__(
+      self,
+      columns: List[str],
+      *,
+      # common args for all ModelHandlers.
+      load_model_args: Optional[Dict[str, Any]] = None,
+      min_batch_size: Optional[int] = None,
+      max_batch_size: Optional[int] = None,
+      large_model: bool = False,
+      **kwargs):
+    self.load_model_args = load_model_args or {}
+    self.min_batch_size = min_batch_size
+    self.max_batch_size = max_batch_size
+    self.large_model = large_model
+    self.columns = columns
+    self.inference_args = kwargs.pop('inference_args', {})
+
+    if kwargs:
+      _LOGGER.warning("Ignoring the following arguments: %s", kwargs.keys())
+
+  # TODO:https://github.com/apache/beam/pull/29564 add set_model_handler method
   @abc.abstractmethod
-  def append_transform(self, transform: BaseOperation):
+  def get_model_handler(self) -> ModelHandler:
     """
-    Append transforms to the ProcessHandler.
+    Return framework specific model handler.
     """
 
+  def get_columns_to_apply(self):
+    return self.columns
+
 
 class MLTransform(beam.PTransform[beam.PCollection[ExampleT],
                                   beam.PCollection[MLTransformOutputT]],
@@ -112,7 +204,7 @@ def __init__(
       *,
       write_artifact_location: Optional[str] = None,
       read_artifact_location: Optional[str] = None,
-      transforms: Optional[Sequence[BaseOperation]] = None):
+      transforms: Optional[List[MLTransformProvider]] = None):
     """
     MLTransform is a Beam PTransform that can be used to apply
     transformations to the data. MLTransform is used to wrap the
@@ -157,9 +249,6 @@ def __init__(
         i-th transform is the output of the (i-1)-th transform. Multi-input
         transforms are not supported yet.
     """
-    if transforms:
-      _ = [self._validate_transform(transform) for transform in transforms]
-
     if read_artifact_location and write_artifact_location:
       raise ValueError(
           'Only one of read_artifact_location or write_artifact_location can '
@@ -173,23 +262,19 @@ def __init__(
     if read_artifact_location:
       artifact_location = read_artifact_location
       artifact_mode = ArtifactMode.CONSUME
+      if transforms:
+        raise ValueError(
+            'Transforms should not be passed in read mode. In read mode, '
+            'the transforms are read from the artifact location.')
+
     else:
       artifact_location = write_artifact_location  # type: ignore[assignment]
       artifact_mode = ArtifactMode.PRODUCE
 
-    # avoid circular import
-    # pylint: disable=wrong-import-order, wrong-import-position
-    from apache_beam.ml.transforms.handlers import TFTProcessHandler
-    # TODO: When new ProcessHandlers(eg: JaxProcessHandler) are introduced,
-    # create a mapping between transforms and ProcessHandler since
-    # ProcessHandler is not exposed to the user.
-    process_handler: ProcessHandler = TFTProcessHandler(
-        artifact_location=artifact_location,
-        artifact_mode=artifact_mode,
-        transforms=transforms)  # type: ignore[arg-type]
-
-    self._process_handler = process_handler
-    self.transforms = transforms
+    self._parent_artifact_location = artifact_location
+
+    self._artifact_mode = artifact_mode
+    self.transforms = transforms or []
     self._counter = Metrics.counter(
         MLTransform, f'BeamML_{self.__class__.__name__}')
 
@@ -209,12 +294,34 @@ def expand(
     Returns:
       A PCollection of MLTransformOutputT type
     """
+    _ = [self._validate_transform(transform) for transform in self.transforms]
+    if self._artifact_mode == ArtifactMode.PRODUCE:
+      ptransform_partitioner = _MLTransformToPTransformMapper(
+          transforms=self.transforms,
+          artifact_location=self._parent_artifact_location,
+          artifact_mode=self._artifact_mode,
+          pipeline_options=pcoll.pipeline.options)
+      ptransform_list = ptransform_partitioner.create_and_save_ptransform_list()
+    else:
+      ptransform_list = (
+          _MLTransformToPTransformMapper.load_transforms_from_artifact_location(
+              self._parent_artifact_location))
+
+      # the saved transforms has artifact mode set to PRODUCE.
+      # set the artifact mode to CONSUME.
+      for i in range(len(ptransform_list)):
+        if hasattr(ptransform_list[i], 'artifact_mode'):
+          ptransform_list[i].artifact_mode = self._artifact_mode
+
+    for ptransform in ptransform_list:
+      pcoll = pcoll | ptransform
+
     _ = (
         pcoll.pipeline
         | "MLTransformMetricsUsage" >> MLTransformMetricsUsage(self))
-    return self._process_handler.process_data(pcoll)
+    return pcoll  # type: ignore[return-value]
 
-  def with_transform(self, transform: BaseOperation):
+  def with_transform(self, transform: MLTransformProvider):
     """
     Add a transform to the MLTransform pipeline.
     Args:
@@ -223,11 +330,11 @@ def with_transform(self, transform: BaseOperation):
       A MLTransform instance.
     """
     self._validate_transform(transform)
-    self._process_handler.append_transform(transform)
+    self.transforms.append(transform)
     return self
 
   def _validate_transform(self, transform):
-    if not isinstance(transform, BaseOperation):
+    if not isinstance(transform, MLTransformProvider):
       raise TypeError(
           'transform must be a subclass of BaseOperation. '
           'Got: %s instead.' % type(transform))
@@ -243,9 +350,7 @@ def _increment_counters():
       # increment for MLTransform.
       self._ml_transform._counter.inc()
       # increment if data processing transforms are passed.
-      transforms = (
-          self._ml_transform.transforms or
-          self._ml_transform._process_handler.transforms)
+      transforms = self._ml_transform.transforms
       if transforms:
         for transform in transforms:
           transform.get_counter().inc()
@@ -254,3 +359,263 @@ def _increment_counters():
         pipeline
         | beam.Create([None])
         | beam.Map(lambda _: _increment_counters()))
+
+
+class _TransformAttributeManager:
+  """
+  Base class used for saving and loading the attributes.
+  """
+  @staticmethod
+  def save_attributes(artifact_location):
+    """
+    Save the attributes to json file using stdlib json.
+    """
+    raise NotImplementedError
+
+  @staticmethod
+  def load_attributes(artifact_location):
+    """
+    Load the attributes from json file.
+    """
+    raise NotImplementedError
+
+
+class _JsonPickleTransformAttributeManager(_TransformAttributeManager):
+  """
+  Use Jsonpickle to save and load the attributes. Here the attributes refer
+  to the list of PTransforms that are used to process the data.
+
+  jsonpickle is used to serialize the PTransforms and save it to a json file and
+  is compatible across python versions.
+  """
+  @staticmethod
+  def _is_remote_path(path):
+    is_gcs = path.find('gs://') != -1
+    # TODO:https://github.com/apache/beam/issues/29356
+    #  Add support for other remote paths.
+    if not is_gcs and path.find('://') != -1:
+      raise RuntimeError(
+          "Artifact locations are currently supported for only available for "
+          "local paths and GCS paths. Got: %s" % path)
+    return is_gcs
+
+  @staticmethod
+  def save_attributes(
+      ptransform_list,
+      artifact_location,
+      **kwargs,
+  ):
+    # if an artifact location is present, instead of overwriting the
+    # existing file, raise an error since the same artifact location
+    # can be used by multiple beam jobs and this could result in undesired
+    # behavior.
+    if FileSystems.exists(FileSystems.join(artifact_location,
+                                           _ATTRIBUTE_FILE_NAME)):
+      raise FileExistsError(
+          "The artifact location %s already exists and contains %s. Please "
+          "specify a different location." %
+          (artifact_location, _ATTRIBUTE_FILE_NAME))
+
+    if _JsonPickleTransformAttributeManager._is_remote_path(artifact_location):
+      temp_dir = tempfile.mkdtemp()
+      temp_json_file = os.path.join(temp_dir, _ATTRIBUTE_FILE_NAME)
+      with open(temp_json_file, 'w+') as f:
+        f.write(jsonpickle.encode(ptransform_list))
+      with open(temp_json_file, 'rb') as f:
+        from apache_beam.runners.dataflow.internal import apiclient
+        _LOGGER.info('Creating artifact location: %s', artifact_location)
+        # pipeline options required to for the client to configure project.
+        options = kwargs.get('options')
+        try:
+          apiclient.DataflowApplicationClient(options=options).stage_file(
+              gcs_or_local_path=artifact_location,
+              file_name=_ATTRIBUTE_FILE_NAME,
+              stream=f,
+              mime_type='application/json')
+        except Exception as exc:
+          if not options:
+            raise RuntimeError(
+                "Failed to create Dataflow client. "
+                "Pipeline options are required to save the attributes."
+                "in the artifact location %s" % artifact_location) from exc
+          raise
+    else:
+      if not FileSystems.exists(artifact_location):
+        FileSystems.mkdirs(artifact_location)
+      # FileSystems.open() fails if the file does not exist.
+      with open(os.path.join(artifact_location, _ATTRIBUTE_FILE_NAME),
+                'w+') as f:
+        f.write(jsonpickle.encode(ptransform_list))
+
+  @staticmethod
+  def load_attributes(artifact_location):
+    with FileSystems.open(os.path.join(artifact_location, _ATTRIBUTE_FILE_NAME),
+                          'rb') as f:
+      return jsonpickle.decode(f.read())
+
+
+_transform_attribute_manager = _JsonPickleTransformAttributeManager
+
+
+class _MLTransformToPTransformMapper:
+  """
+  This class takes in a list of data processing transforms compatible to be
+  wrapped around MLTransform and returns a list of PTransforms that are used to
+  run the data processing transforms.
+
+  The _MLTransformToPTransformMapper is responsible for loading and saving the
+  PTransforms or attributes of PTransforms to the artifact location to seal
+  the gap between the training and inference pipelines.
+  """
+  def __init__(
+      self,
+      transforms: List[MLTransformProvider],
+      artifact_location: str,
+      artifact_mode: str = ArtifactMode.PRODUCE,
+      pipeline_options: Optional[PipelineOptions] = None,
+  ):
+    self.transforms = transforms
+    self._parent_artifact_location = artifact_location
+    self.artifact_mode = artifact_mode
+    self.pipeline_options = pipeline_options
+
+  def create_and_save_ptransform_list(self):
+    ptransform_list = self.create_ptransform_list()
+    self.save_transforms_in_artifact_location(ptransform_list)
+    return ptransform_list
+
+  def create_ptransform_list(self):
+    previous_ptransform_type = None
+    current_ptransform = None
+    ptransform_list = []
+    for transform in self.transforms:
+      if not isinstance(transform, MLTransformProvider):
+        raise RuntimeError(
+            'Transforms must be instances of MLTransformProvider and '
+            'implement get_ptransform_for_processing() method.')
+      # for each instance of PTransform, create a new artifact location
+      current_ptransform = transform.get_ptransform_for_processing(
+          artifact_location=os.path.join(
+              self._parent_artifact_location, uuid.uuid4().hex[:6]),
+          artifact_mode=self.artifact_mode)
+      append_transform = hasattr(current_ptransform, 'append_transform')
+      if (type(current_ptransform) !=
+          previous_ptransform_type) or not append_transform:
+        ptransform_list.append(current_ptransform)
+        previous_ptransform_type = type(current_ptransform)
+      # If different PTransform is appended to the list and the PTransform
+      # supports append_transform, append the transform to the PTransform.
+      if append_transform:
+        ptransform_list[-1].append_transform(transform)
+    return ptransform_list
+
+  def save_transforms_in_artifact_location(self, ptransform_list):
+    """
+    Save the ptransform references to json file.
+    """
+    _transform_attribute_manager.save_attributes(
+        ptransform_list=ptransform_list,
+        artifact_location=self._parent_artifact_location,
+        options=self.pipeline_options)
+
+  @staticmethod
+  def load_transforms_from_artifact_location(artifact_location):
+    return _transform_attribute_manager.load_attributes(artifact_location)
+
+
+class _TextEmbeddingHandler(ModelHandler):
+  """
+  A ModelHandler intended to be work on list[dict[str, str]] inputs.
+
+  The inputs to the model handler are expected to be a list of dicts.
+
+  For example, if the original mode is used with RunInference to take a
+  PCollection[E] to a PCollection[P], this ModelHandler would take a
+  PCollection[Dict[str, E]] to a PCollection[Dict[str, P]].
+
+  _TextEmbeddingHandler will accept an EmbeddingsManager instance, which
+  contains the details of the model to be loaded and the inference_fn to be
+  used. The purpose of _TextEmbeddingHandler is to generate embeddings for
+  text inputs using the EmbeddingsManager instance.
+
+  If the input is not a text column, a RuntimeError will be raised.
+
+  This is an internal class and offers no backwards compatibility guarantees.
+
+  Args:
+    embeddings_manager: An EmbeddingsManager instance.
+  """
+  def __init__(self, embeddings_manager: EmbeddingsManager):
+    self.embedding_config = embeddings_manager
+    self._underlying = self.embedding_config.get_model_handler()
+    self.columns = self.embedding_config.get_columns_to_apply()
+
+  def load_model(self):
+    model = self._underlying.load_model()
+    return model
+
+  def _validate_column_data(self, batch):
+    if not isinstance(batch[0], (str, bytes)):
+      raise TypeError(
+          'Embeddings can only be generated on Dict[str, str].'
+          f'Got Dict[str, {type(batch[0])}] instead.')
+
+  def _validate_batch(self, batch: Sequence[Dict[str, List[str]]]):
+    if not batch or not isinstance(batch[0], dict):
+      raise TypeError(
+          'Expected data to be dicts, got '
+          f'{type(batch[0])} instead.')
+
+  def _process_batch(
+      self,
+      dict_batch: Dict[str, List[Any]],
+      model: ModelT,
+      inference_args: Optional[Dict[str, Any]]) -> Dict[str, List[Any]]:
+    result: Dict[str, List[Any]] = collections.defaultdict(list)
+    for key, batch in dict_batch.items():
+      if key in self.columns:
+        self._validate_column_data(batch)
+        prediction = self._underlying.run_inference(
+            batch, model, inference_args)
+        if isinstance(prediction, np.ndarray):
+          prediction = prediction.tolist()
+          result[key] = prediction  # type: ignore[assignment]
+        else:
+          result[key] = prediction  # type: ignore[assignment]
+      else:
+        result[key] = batch
+    return result
+
+  def run_inference(
+      self,
+      batch: Sequence[Dict[str, List[str]]],
+      model: ModelT,
+      inference_args: Optional[Dict[str, Any]] = None,
+  ) -> List[Dict[str, Union[List[float], List[str]]]]:
+    """
+    Runs inference on a batch of text inputs. The inputs are expected to be
+    a list of dicts. Each dict should have the same keys, and the shape
+    should be of the same size for a single key across the batch.
+    """
+    self._validate_batch(batch)
+    dict_batch = _convert_list_of_dicts_to_dict_of_lists(list_of_dicts=batch)
+    transformed_batch = self._process_batch(dict_batch, model, inference_args)
+    return _convert_dict_of_lists_to_lists_of_dict(
+        dict_of_lists=transformed_batch,
+    )
+
+  def get_metrics_namespace(self) -> str:
+    return (
+        self._underlying.get_metrics_namespace() or
+        'BeamML_TextEmbeddingHandler')
+
+  def batch_elements_kwargs(self) -> Mapping[str, Any]:
+    batch_sizes_map = {}
+    if self.embedding_config.max_batch_size:
+      batch_sizes_map['max_batch_size'] = self.embedding_config.max_batch_size
+    if self.embedding_config.min_batch_size:
+      batch_sizes_map['min_batch_size'] = self.embedding_config.min_batch_size
+    return (self._underlying.batch_elements_kwargs() or batch_sizes_map)
+
+  def validate_inference_args(self, _):
+    pass
diff --git a/sdks/python/apache_beam/ml/transforms/base_test.py b/sdks/python/apache_beam/ml/transforms/base_test.py
index 2e447964541b..d8121e29b43a 100644
--- a/sdks/python/apache_beam/ml/transforms/base_test.py
+++ b/sdks/python/apache_beam/ml/transforms/base_test.py
@@ -16,11 +16,16 @@
 #
 # pytype: skip-file
 
+import os
 import shutil
 import tempfile
 import typing
 import unittest
+from typing import Any
+from typing import Dict
 from typing import List
+from typing import Optional
+from typing import Sequence
 
 import numpy as np
 from parameterized import param
@@ -28,28 +33,36 @@
 
 import apache_beam as beam
 from apache_beam.metrics.metric import MetricsFilter
+from apache_beam.ml.inference.base import ModelHandler
+from apache_beam.ml.inference.base import RunInference
+from apache_beam.ml.transforms import base
 from apache_beam.testing.util import assert_that
 from apache_beam.testing.util import equal_to
 
 # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
 try:
-  from apache_beam.ml.transforms import base
   from apache_beam.ml.transforms import tft
+  from apache_beam.ml.transforms.handlers import TFTProcessHandler
   from apache_beam.ml.transforms.tft import TFTOperation
 except ImportError:
   tft = None  # type: ignore
 
-if tft is None:
-  raise unittest.SkipTest('tensorflow_transform is not installed')
+try:
 
+  class _FakeOperation(TFTOperation):
+    def __init__(self, name, *args, **kwargs):
+      super().__init__(*args, **kwargs)
+      self.name = name
 
-class _FakeOperation(TFTOperation):
-  def __init__(self, name, *args, **kwargs):
-    super().__init__(*args, **kwargs)
-    self.name = name
+    def apply_transform(self, inputs, output_column_name, **kwargs):
+      return {output_column_name: inputs}
+except:  # pylint: disable=bare-except
+  pass
 
-  def apply_transform(self, inputs, output_column_name, **kwargs):
-    return {output_column_name: inputs}
+try:
+  from apache_beam.runners.dataflow.internal import apiclient
+except ImportError:
+  apiclient = None  # type: ignore
 
 
 class BaseMLTransformTest(unittest.TestCase):
@@ -59,6 +72,7 @@ def setUp(self) -> None:
   def tearDown(self):
     shutil.rmtree(self.artifact_location)
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transform_appends_transforms_to_process_handler_correctly(self):
     fake_fn_1 = _FakeOperation(name='fake_fn_1', columns=['x'])
     transforms = [fake_fn_1]
@@ -67,12 +81,11 @@ def test_ml_transform_appends_transforms_to_process_handler_correctly(self):
     ml_transform = ml_transform.with_transform(
         transform=_FakeOperation(name='fake_fn_2', columns=['x']))
 
-    self.assertEqual(len(ml_transform._process_handler.transforms), 2)
-    self.assertEqual(
-        ml_transform._process_handler.transforms[0].name, 'fake_fn_1')
-    self.assertEqual(
-        ml_transform._process_handler.transforms[1].name, 'fake_fn_2')
+    self.assertEqual(len(ml_transform.transforms), 2)
+    self.assertEqual(ml_transform.transforms[0].name, 'fake_fn_1')
+    self.assertEqual(ml_transform.transforms[1].name, 'fake_fn_2')
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transform_on_dict(self):
     transforms = [tft.ScaleTo01(columns=['x'])]
     data = [{'x': 1}, {'x': 2}]
@@ -91,6 +104,7 @@ def test_ml_transform_on_dict(self):
       assert_that(
           actual_output, equal_to(expected_output, equals_fn=np.array_equal))
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transform_on_list_dict(self):
     transforms = [tft.ScaleTo01(columns=['x'])]
     data = [{'x': [1, 2, 3]}, {'x': [4, 5, 6]}]
@@ -162,6 +176,7 @@ def test_ml_transform_on_list_dict(self):
           },
       ),
   ])
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transform_dict_output_pcoll_schema(
       self, input_data, input_types, expected_dtype):
     transforms = [tft.ScaleTo01(columns=['x'])]
@@ -178,6 +193,7 @@ def test_ml_transform_dict_output_pcoll_schema(
         if name in expected_dtype:
           self.assertEqual(expected_dtype[name], typ)
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transform_fail_for_non_global_windows_in_produce_mode(self):
     transforms = [tft.ScaleTo01(columns=['x'])]
     with beam.Pipeline() as p:
@@ -193,6 +209,7 @@ def test_ml_transform_fail_for_non_global_windows_in_produce_mode(self):
                 write_artifact_location=self.artifact_location,
             ))
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transform_on_multiple_columns_single_transform(self):
     transforms = [tft.ScaleTo01(columns=['x', 'y'])]
     data = [{'x': [1, 2, 3], 'y': [1.0, 10.0, 20.0]}]
@@ -217,6 +234,7 @@ def test_ml_transform_on_multiple_columns_single_transform(self):
           equal_to(expected_output_y, equals_fn=np.array_equal),
           label='y')
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_ml_transforms_on_multiple_columns_multiple_transforms(self):
     transforms = [
         tft.ScaleTo01(columns=['x']),
@@ -245,6 +263,7 @@ def test_ml_transforms_on_multiple_columns_multiple_transforms(self):
           equal_to(expected_output_y, equals_fn=np.array_equal),
           label='actual_output_y')
 
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
   def test_mltransform_with_counter(self):
     transforms = [
         tft.ComputeAndApplyVocabulary(columns=['y']),
@@ -269,6 +288,305 @@ def test_mltransform_with_counter(self):
     self.assertEqual(
         result.metrics().query(mltransform_counter)['counters'][0].result, 1)
 
+  def test_non_ptransfrom_provider_class_to_mltransform(self):
+    class Add:
+      def __call__(self, x):
+        return x + 1
+
+    with self.assertRaisesRegex(TypeError, 'transform must be a subclass of'):
+      with beam.Pipeline() as p:
+        _ = (
+            p
+            | beam.Create([{
+                'x': 1
+            }])
+            | base.MLTransform(
+                write_artifact_location=self.artifact_location).with_transform(
+                    Add()))
+
+  def test_read_mode_with_transforms(self):
+    with self.assertRaises(ValueError):
+      _ = base.MLTransform(
+          # fake callable
+          transforms=[lambda x: x],
+          read_artifact_location=self.artifact_location)
+
+
+class FakeModel:
+  def __call__(self, example: List[str]) -> List[str]:
+    for i in range(len(example)):
+      example[i] = example[i][::-1]
+    return example
+
+
+class FakeModelHandler(ModelHandler):
+  def run_inference(
+      self,
+      batch: Sequence[str],
+      model: Any,
+      inference_args: Optional[Dict[str, Any]] = None):
+    return model(batch)
+
+  def load_model(self):
+    return FakeModel()
+
+
+class FakeEmbeddingsManager(base.EmbeddingsManager):
+  def __init__(self, columns):
+    super().__init__(columns=columns)
+
+  def get_model_handler(self) -> ModelHandler:
+    return FakeModelHandler()
+
+  def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
+    return (RunInference(model_handler=base._TextEmbeddingHandler(self)))
+
+
+class TextEmbeddingHandlerTest(unittest.TestCase):
+  def setUp(self) -> None:
+    self.embedding_conig = FakeEmbeddingsManager(columns=['x'])
+    self.artifact_location = tempfile.mkdtemp()
+
+  def tearDown(self) -> None:
+    shutil.rmtree(self.artifact_location)
+
+  def test_handler_with_incompatible_datatype(self):
+    text_handler = base._TextEmbeddingHandler(
+        embeddings_manager=self.embedding_conig)
+    data = [
+        ('x', 1),
+        ('x', 2),
+        ('x', 3),
+    ]
+    with self.assertRaises(TypeError):
+      text_handler.run_inference(data, None, None)
+
+  def test_handler_with_dict_inputs(self):
+    data = [
+        {
+            'x': "Hello world"
+        },
+        {
+            'x': "Apache Beam"
+        },
+    ]
+    expected_data = [{key: value[::-1]
+                      for key, value in d.items()} for d in data]
+    with beam.Pipeline() as p:
+      result = (
+          p
+          | beam.Create(data)
+          | base.MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  self.embedding_conig))
+      assert_that(
+          result,
+          equal_to(expected_data),
+      )
+
+  def test_handler_with_batch_sizes(self):
+    self.embedding_conig.max_batch_size = 100
+    self.embedding_conig.min_batch_size = 10
+    data = [
+        {
+            'x': "Hello world"
+        },
+        {
+            'x': "Apache Beam"
+        },
+    ] * 100
+    expected_data = [{key: value[::-1]
+                      for key, value in d.items()} for d in data]
+    with beam.Pipeline() as p:
+      result = (
+          p
+          | beam.Create(data)
+          | base.MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  self.embedding_conig))
+      assert_that(
+          result,
+          equal_to(expected_data),
+      )
+
+  def test_handler_on_multiple_columns(self):
+    self.embedding_conig.columns = ['x', 'y']
+    data = [
+        {
+            'x': "Hello world", 'y': "Apache Beam", 'z': 'unchanged'
+        },
+        {
+            'x': "Apache Beam", 'y': "Hello world", 'z': 'unchanged'
+        },
+    ]
+    self.embedding_conig.columns = ['x', 'y']
+    expected_data = [{
+        key: (value[::-1] if key in self.embedding_conig.columns else value)
+        for key,
+        value in d.items()
+    } for d in data]
+    with beam.Pipeline() as p:
+      result = (
+          p
+          | beam.Create(data)
+          | base.MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  self.embedding_conig))
+      assert_that(
+          result,
+          equal_to(expected_data),
+      )
+
+  def test_handler_with_list_data(self):
+    data = [{
+        'x': ['Hello world', 'Apache Beam'],
+    }, {
+        'x': ['Apache Beam', 'Hello world'],
+    }]
+    with self.assertRaises(TypeError):
+      with beam.Pipeline() as p:
+        _ = (
+            p
+            | beam.Create(data)
+            | base.MLTransform(
+                write_artifact_location=self.artifact_location).with_transform(
+                    self.embedding_conig))
+
+
+class TestUtilFunctions(unittest.TestCase):
+  def test_list_of_dicts_to_dict_of_lists_normal(self):
+    input_list = [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}]
+    expected_output = {'a': [1, 3], 'b': [2, 4]}
+    self.assertEqual(
+        base._convert_list_of_dicts_to_dict_of_lists(input_list),
+        expected_output)
+
+  def test_list_of_dicts_to_dict_of_lists_on_list_inputs(self):
+    input_list = [{'a': [1, 2, 10], 'b': 3}, {'a': [1], 'b': 5}]
+    expected_output = {'a': [[1, 2, 10], [1]], 'b': [3, 5]}
+    self.assertEqual(
+        base._convert_list_of_dicts_to_dict_of_lists(input_list),
+        expected_output)
+
+  def test_dict_of_lists_to_lists_of_dict_normal(self):
+    input_dict = {'a': [1, 3], 'b': [2, 4]}
+    expected_output = [{'a': 1, 'b': 2}, {'a': 3, 'b': 4}]
+    self.assertEqual(
+        base._convert_dict_of_lists_to_lists_of_dict(input_dict),
+        expected_output)
+
+  def test_dict_of_lists_to_lists_of_dict_unequal_length(self):
+    input_dict = {'a': [1, 3], 'b': [2]}
+    with self.assertRaises(AssertionError):
+      base._convert_dict_of_lists_to_lists_of_dict(input_dict)
+
+
+class TestJsonPickleTransformAttributeManager(unittest.TestCase):
+  def setUp(self):
+    self.attribute_manager = base._transform_attribute_manager
+    self.artifact_location = tempfile.mkdtemp()
+
+  def tearDown(self):
+    shutil.rmtree(self.artifact_location)
+
+  @unittest.skipIf(tft is None, 'tft module is not installed.')
+  def test_save_tft_process_handler(self):
+    transforms = [
+        tft.ScaleTo01(columns=['x']),
+        tft.ComputeAndApplyVocabulary(columns=['y'])
+    ]
+    process_handler = TFTProcessHandler(
+        transforms=transforms,
+        artifact_location=self.artifact_location,
+    )
+    self.attribute_manager.save_attributes(
+        ptransform_list=[process_handler],
+        artifact_location=self.artifact_location,
+    )
+
+    files = os.listdir(self.artifact_location)
+    self.assertTrue(len(files) == 1)
+    self.assertTrue(files[0] == base._ATTRIBUTE_FILE_NAME)
+
+  def test_save_run_inference(self):
+    self.attribute_manager.save_attributes(
+        ptransform_list=[RunInference(model_handler=FakeModelHandler())],
+        artifact_location=self.artifact_location,
+    )
+    files = os.listdir(self.artifact_location)
+    self.assertTrue(len(files) == 1)
+    self.assertTrue(files[0] == base._ATTRIBUTE_FILE_NAME)
+
+  def test_save_and_load_run_inference(self):
+    ptransform_list = [RunInference(model_handler=FakeModelHandler())]
+    self.attribute_manager.save_attributes(
+        ptransform_list=ptransform_list,
+        artifact_location=self.artifact_location,
+    )
+    loaded_ptransform_list = self.attribute_manager.load_attributes(
+        artifact_location=self.artifact_location,
+    )
+
+    self.assertTrue(len(loaded_ptransform_list) == len(ptransform_list))
+    self.assertListEqual(
+        list(loaded_ptransform_list[0].__dict__.keys()),
+        list(ptransform_list[0].__dict__.keys()))
+
+    get_keys = lambda x: list(x.__dict__.keys())
+    for i, transform in enumerate(ptransform_list):
+      self.assertListEqual(
+          get_keys(transform), get_keys(loaded_ptransform_list[i]))
+      if hasattr(transform, 'model_handler'):
+        model_handler = transform.model_handler
+        loaded_model_handler = loaded_ptransform_list[i].model_handler
+        self.assertListEqual(
+            get_keys(model_handler), get_keys(loaded_model_handler))
+
+  def test_mltransform_to_ptransform_wrapper(self):
+    transforms = [
+        FakeEmbeddingsManager(columns=['x']),
+        FakeEmbeddingsManager(columns=['y', 'z']),
+    ]
+    ptransform_mapper = base._MLTransformToPTransformMapper(
+        transforms=transforms,
+        artifact_location=self.artifact_location,
+        artifact_mode=None)
+
+    ptransform_list = ptransform_mapper.create_ptransform_list()
+    self.assertTrue(len(ptransform_list) == 2)
+
+    self.assertEqual(type(ptransform_list[0]), RunInference)
+    expected_columns = [['x'], ['y', 'z']]
+    for i in range(len(ptransform_list)):
+      self.assertEqual(type(ptransform_list[i]), RunInference)
+      self.assertEqual(
+          type(ptransform_list[i]._model_handler), base._TextEmbeddingHandler)
+      self.assertEqual(
+          ptransform_list[i]._model_handler.columns, expected_columns[i])
+
+  @unittest.skipIf(apiclient is None, 'apache_beam[gcp] is not installed.')
+  def test_with_gcs_location_with_none_options(self):
+    path = 'gs://fake_path'
+    with self.assertRaises(RuntimeError):
+      self.attribute_manager.save_attributes(
+          ptransform_list=[], artifact_location=path, options=None)
+    with self.assertRaises(RuntimeError):
+      self.attribute_manager.save_attributes(
+          ptransform_list=[], artifact_location=path)
+
+  def test_with_same_local_artifact_location(self):
+    artifact_location = self.artifact_location
+    attribute_manager = base._JsonPickleTransformAttributeManager()
+
+    ptransform_list = [RunInference(model_handler=FakeModelHandler())]
+
+    attribute_manager.save_attributes(
+        ptransform_list, artifact_location=artifact_location)
+
+    with self.assertRaises(FileExistsError):
+      attribute_manager.save_attributes([lambda x: x],
+                                        artifact_location=artifact_location)
+
 
 if __name__ == '__main__':
   unittest.main()
diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/__init__.py b/sdks/python/apache_beam/ml/transforms/embeddings/__init__.py
new file mode 100644
index 000000000000..bda6256b79ef
--- /dev/null
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/__init__.py
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# TODO: Add dead letter queue for RunInference transforms.
+
+"""
+This module contains embedding configs that can be used to generate
+embeddings using MLTransform.
+"""
diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
new file mode 100644
index 000000000000..e979296b0b83
--- /dev/null
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface.py
@@ -0,0 +1,131 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__all__ = ["SentenceTransformerEmbeddings"]
+
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Optional
+from typing import Sequence
+
+import apache_beam as beam
+from apache_beam.ml.inference.base import ModelHandler
+from apache_beam.ml.inference.base import RunInference
+from apache_beam.ml.transforms.base import EmbeddingsManager
+from apache_beam.ml.transforms.base import _TextEmbeddingHandler
+from sentence_transformers import SentenceTransformer
+
+
+# TODO: https://github.com/apache/beam/issues/29621
+# Use HuggingFaceModelHandlerTensor once the import issue is fixed.
+# Right now, the hugging face model handler import torch and tensorflow
+# at the same time, which adds too much weigth to the container unnecessarily.
+class _SentenceTransformerModelHandler(ModelHandler):
+  """
+  Note: Intended for internal use and guarantees no backwards compatibility.
+  """
+  def __init__(
+      self,
+      model_name: str,
+      model_class: Callable,
+      load_model_args: Optional[dict] = None,
+      min_batch_size: Optional[int] = None,
+      max_batch_size: Optional[int] = None,
+      max_seq_length: Optional[int] = None,
+      large_model: bool = False,
+      **kwargs):
+    self._max_seq_length = max_seq_length
+    self.model_name = model_name
+    self._model_class = model_class
+    self._load_model_args = load_model_args
+    self._min_batch_size = min_batch_size
+    self._max_batch_size = max_batch_size
+    self._large_model = large_model
+    self._kwargs = kwargs
+
+  def run_inference(
+      self,
+      batch: Sequence[str],
+      model: SentenceTransformer,
+      inference_args: Optional[Dict[str, Any]] = None,
+  ):
+    inference_args = inference_args or {}
+    return model.encode(batch, **inference_args)
+
+  def load_model(self):
+    model = self._model_class(self.model_name, **self._load_model_args)
+    if self._max_seq_length:
+      model.max_seq_length = self._max_seq_length
+    return model
+
+  def share_model_across_processes(self) -> bool:
+    return self._large_model
+
+  def batch_elements_kwargs(self) -> Mapping[str, Any]:
+    batch_sizes = {}
+    if self._min_batch_size:
+      batch_sizes["min_batch_size"] = self._min_batch_size
+    if self._max_batch_size:
+      batch_sizes["max_batch_size"] = self._max_batch_size
+    return batch_sizes
+
+
+class SentenceTransformerEmbeddings(EmbeddingsManager):
+  def __init__(
+      self,
+      model_name: str,
+      columns: List[str],
+      max_seq_length: Optional[int] = None,
+      **kwargs):
+    """
+    Embedding config for sentence-transformers. This config can be used with
+    MLTransform to embed text data. Models are loaded using the RunInference
+    PTransform with the help of ModelHandler.
+
+    Args:
+      model_name: Name of the model to use. The model should be hosted on
+        HuggingFace Hub or compatible with sentence_transformers.
+      columns: List of columns to be embedded.
+      max_seq_length: Max sequence length to use for the model if applicable.
+      min_batch_size: The minimum batch size to be used for inference.
+      max_batch_size: The maximum batch size to be used for inference.
+      large_model: Whether to share the model across processes.
+    """
+    super().__init__(columns, **kwargs)
+    self.model_name = model_name
+    self.max_seq_length = max_seq_length
+
+  def get_model_handler(self):
+    return _SentenceTransformerModelHandler(
+        model_class=SentenceTransformer,
+        max_seq_length=self.max_seq_length,
+        model_name=self.model_name,
+        load_model_args=self.load_model_args,
+        min_batch_size=self.min_batch_size,
+        max_batch_size=self.max_batch_size,
+        large_model=self.large_model)
+
+  def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
+    # wrap the model handler in a _TextEmbeddingHandler since
+    # the SentenceTransformerEmbeddings works on text input data.
+    return (
+        RunInference(
+            model_handler=_TextEmbeddingHandler(self),
+            inference_args=self.inference_args,
+        ))
diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
new file mode 100644
index 000000000000..779a6daf8f3c
--- /dev/null
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/huggingface_test.py
@@ -0,0 +1,278 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+import uuid
+
+import numpy as np
+from parameterized import parameterized
+
+import apache_beam as beam
+from apache_beam.ml.inference.base import RunInference
+from apache_beam.ml.transforms import base
+from apache_beam.ml.transforms.base import MLTransform
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+
+# pylint: disable=ungrouped-imports
+try:
+  from apache_beam.ml.transforms.embeddings.huggingface import SentenceTransformerEmbeddings
+  import torch
+except ImportError:
+  SentenceTransformerEmbeddings = None  # type: ignore
+
+# pylint: disable=ungrouped-imports
+try:
+  import tensorflow_transform as tft
+  from apache_beam.ml.transforms.tft import ScaleTo01
+except ImportError:
+  tft = None
+
+test_query = "This is a test"
+test_query_column = "feature_1"
+DEFAULT_MODEL_NAME = "sentence-transformers/all-mpnet-base-v2"
+_parameterized_inputs = [
+    ([{
+        test_query_column: 'That is a happy person'
+    }, {
+        test_query_column: 'That is a very happy person'
+    }],
+     'thenlper/gte-base', [0.11, 0.11]),
+    ([{
+        test_query_column: test_query,
+    }], DEFAULT_MODEL_NAME, [0.13]),
+    (
+        [{
+            test_query_column: 'query: how much protein should a female eat',
+        },
+         {
+             test_query_column: (
+                 "passage: As a general guideline, the CDC's "
+                 "average requirement of protein for women "
+                 "ages 19 to 70 is 46 grams per day. But, "
+                 "as you can see from this chart, you'll need "
+                 "to increase that if you're expecting or training"
+                 " for a marathon. Check out the chart below "
+                 "to see how much protein "
+                 "you should be eating each day.")
+         }],
+        'intfloat/e5-base-v2',
+        # this model requires inputs to be specified as query: and passage:
+        [0.1, 0.1]),
+]
+
+
+@unittest.skipIf(
+    SentenceTransformerEmbeddings is None,
+    'sentence-transformers is not installed.')
+class SentenceTrasformerEmbeddingsTest(unittest.TestCase):
+  def setUp(self) -> None:
+    self.artifact_location = tempfile.mkdtemp(prefix='sentence_transformers_')
+    # this bucket has TTL and will be deleted periodically
+    self.gcs_artifact_location = os.path.join(
+        'gs://temp-storage-for-perf-tests/sentence_transformers',
+        uuid.uuid4().hex)
+
+  def tearDown(self) -> None:
+    shutil.rmtree(self.artifact_location)
+
+  def test_sentence_transformer_embeddings(self):
+    model_name = DEFAULT_MODEL_NAME
+    embedding_config = SentenceTransformerEmbeddings(
+        model_name=model_name, columns=[test_query_column])
+    with beam.Pipeline() as pipeline:
+      result_pcoll = (
+          pipeline
+          | "CreateData" >> beam.Create([{
+              test_query_column: test_query
+          }])
+          | "MLTransform" >> MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  embedding_config))
+
+      def assert_element(element):
+        assert len(element[test_query_column]) == 768
+
+      _ = (result_pcoll | beam.Map(assert_element))
+
+  @unittest.skipIf(tft is None, 'Tensorflow Transform is not installed.')
+  def test_embeddings_with_scale_to_0_1(self):
+    model_name = DEFAULT_MODEL_NAME
+    embedding_config = SentenceTransformerEmbeddings(
+        model_name=model_name,
+        columns=[test_query_column],
+    )
+    with beam.Pipeline() as pipeline:
+      transformed_pcoll = (
+          pipeline
+          | "CreateData" >> beam.Create([{
+              test_query_column: test_query
+          }])
+          | "MLTransform" >> MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  embedding_config).with_transform(
+                      ScaleTo01(columns=[test_query_column])))
+
+      def assert_element(element):
+        assert max(element.feature_1) == 1
+
+      _ = (transformed_pcoll | beam.Map(assert_element))
+
+  @parameterized.expand(_parameterized_inputs)
+  def test_embeddings_with_read_artifact_location(
+      self, inputs, model_name, output):
+    embedding_config = SentenceTransformerEmbeddings(
+        model_name=model_name, columns=[test_query_column])
+
+    with beam.Pipeline() as p:
+      result_pcoll = (
+          p
+          | "CreateData" >> beam.Create(inputs)
+          | "MLTransform" >> MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  embedding_config))
+      max_ele_pcoll = (
+          result_pcoll
+          | beam.Map(lambda x: round(max(x[test_query_column]), 2)))
+
+      assert_that(max_ele_pcoll, equal_to(output))
+
+    with beam.Pipeline() as p:
+      result_pcoll = (
+          p
+          | "CreateData" >> beam.Create(inputs)
+          | "MLTransform" >>
+          MLTransform(read_artifact_location=self.artifact_location))
+      max_ele_pcoll = (
+          result_pcoll
+          | beam.Map(lambda x: round(max(x[test_query_column]), 2)))
+
+      assert_that(max_ele_pcoll, equal_to(output))
+
+  def test_sentence_transformer_with_int_data_types(self):
+    model_name = DEFAULT_MODEL_NAME
+    embedding_config = SentenceTransformerEmbeddings(
+        model_name=model_name, columns=[test_query_column])
+    with self.assertRaises(TypeError):
+      with beam.Pipeline() as pipeline:
+        _ = (
+            pipeline
+            | "CreateData" >> beam.Create([{
+                test_query_column: 1
+            }])
+            | "MLTransform" >> MLTransform(
+                write_artifact_location=self.artifact_location).with_transform(
+                    embedding_config))
+
+  @parameterized.expand(_parameterized_inputs)
+  def test_with_gcs_artifact_location(self, inputs, model_name, output):
+    embedding_config = SentenceTransformerEmbeddings(
+        model_name=model_name, columns=[test_query_column])
+
+    with beam.Pipeline() as p:
+      result_pcoll = (
+          p
+          | "CreateData" >> beam.Create(inputs)
+          | "MLTransform" >>
+          MLTransform(write_artifact_location=self.gcs_artifact_location
+                      ).with_transform(embedding_config))
+      max_ele_pcoll = (
+          result_pcoll
+          | beam.Map(lambda x: round(np.max(x[test_query_column]), 2)))
+
+      assert_that(max_ele_pcoll, equal_to(output))
+
+    with beam.Pipeline() as p:
+      result_pcoll = (
+          p
+          | "CreateData" >> beam.Create(inputs)
+          | "MLTransform" >>
+          MLTransform(read_artifact_location=self.gcs_artifact_location))
+      max_ele_pcoll = (
+          result_pcoll
+          | beam.Map(lambda x: round(np.max(x[test_query_column]), 2)))
+
+      assert_that(max_ele_pcoll, equal_to(output))
+
+  def test_embeddings_with_inference_args(self):
+    model_name = DEFAULT_MODEL_NAME
+
+    inference_args = {'convert_to_numpy': False}
+    embedding_config = SentenceTransformerEmbeddings(
+        model_name=model_name,
+        columns=[test_query_column],
+        inference_args=inference_args)
+    with beam.Pipeline() as pipeline:
+      result_pcoll = (
+          pipeline
+          | "CreateData" >> beam.Create([{
+              test_query_column: test_query
+          }])
+          | "MLTransform" >> MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  embedding_config))
+
+      def assert_element(element):
+        assert type(element) == torch.Tensor
+
+      _ = (
+          result_pcoll
+          | beam.Map(lambda x: x[test_query_column])
+          | beam.Map(assert_element))
+
+  def test_mltransform_to_ptransform_with_sentence_transformer(self):
+    model_name = ''
+    transforms = [
+        SentenceTransformerEmbeddings(columns=['x'], model_name=model_name),
+        SentenceTransformerEmbeddings(
+            columns=['y', 'z'], model_name=model_name)
+    ]
+    ptransform_mapper = base._MLTransformToPTransformMapper(
+        transforms=transforms,
+        artifact_location=self.artifact_location,
+        artifact_mode=None)
+
+    ptransform_list = ptransform_mapper.create_and_save_ptransform_list()
+    self.assertTrue(len(ptransform_list) == 2)
+
+    self.assertEqual(type(ptransform_list[0]), RunInference)
+    expected_columns = [['x'], ['y', 'z']]
+    for i in range(len(ptransform_list)):
+      self.assertEqual(type(ptransform_list[i]), RunInference)
+      self.assertEqual(
+          type(ptransform_list[i]._model_handler), base._TextEmbeddingHandler)
+      self.assertEqual(
+          ptransform_list[i]._model_handler.columns, expected_columns[i])
+      self.assertEqual(
+          ptransform_list[i]._model_handler._underlying.model_name, model_name)
+    ptransform_list = (
+        base._MLTransformToPTransformMapper.
+        load_transforms_from_artifact_location(self.artifact_location))
+    for i in range(len(ptransform_list)):
+      self.assertEqual(type(ptransform_list[i]), RunInference)
+      self.assertEqual(
+          type(ptransform_list[i]._model_handler), base._TextEmbeddingHandler)
+      self.assertEqual(
+          ptransform_list[i]._model_handler.columns, expected_columns[i])
+      self.assertEqual(
+          ptransform_list[i]._model_handler._underlying.model_name, model_name)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py
new file mode 100644
index 000000000000..1f4c1577eb79
--- /dev/null
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai.py
@@ -0,0 +1,154 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Vertex AI Python SDK is required for this module.
+# Follow https://cloud.google.com/vertex-ai/docs/python-sdk/use-vertex-ai-python-sdk # pylint: disable=line-too-long
+# to install Vertex AI Python SDK.
+
+from typing import Any
+from typing import Dict
+from typing import Iterable
+from typing import List
+from typing import Optional
+from typing import Sequence
+
+from google.auth.credentials import Credentials
+
+import apache_beam as beam
+import vertexai
+from apache_beam.ml.inference.base import ModelHandler
+from apache_beam.ml.inference.base import RunInference
+from apache_beam.ml.transforms.base import EmbeddingsManager
+from apache_beam.ml.transforms.base import _TextEmbeddingHandler
+from vertexai.language_models import TextEmbeddingInput
+from vertexai.language_models import TextEmbeddingModel
+
+__all__ = ["VertexAITextEmbeddings"]
+
+DEFAULT_TASK_TYPE = "RETRIEVAL_DOCUMENT"
+# TODO: https://github.com/apache/beam/issues/29356
+# Can this list be automatically pulled from Vertex SDK?
+TASK_TYPE_INPUTS = [
+    "RETRIEVAL_DOCUMENT",
+    "RETRIEVAL_QUERY",
+    "SEMANTIC_SIMILARITY",
+    "CLASSIFICATION",
+    "CLUSTERING"
+]
+_BATCH_SIZE = 5  # Vertex AI limits requests to 5 at a time.
+
+
+class _VertexAITextEmbeddingHandler(ModelHandler):
+  """
+  Note: Intended for internal use and guarantees no backwards compatibility.
+  """
+  def __init__(
+      self,
+      model_name: str,
+      title: Optional[str] = None,
+      task_type: str = DEFAULT_TASK_TYPE,
+      project: Optional[str] = None,
+      location: Optional[str] = None,
+      credentials: Optional[Credentials] = None,
+  ):
+    vertexai.init(project=project, location=location, credentials=credentials)
+    self.model_name = model_name
+    if task_type not in TASK_TYPE_INPUTS:
+      raise ValueError(
+          f"task_type must be one of {TASK_TYPE_INPUTS}, got {task_type}")
+    self.task_type = task_type
+    self.title = title
+
+  def run_inference(
+      self,
+      batch: Sequence[str],
+      model: Any,
+      inference_args: Optional[Dict[str, Any]] = None,
+  ) -> Iterable:
+    embeddings = []
+    batch_size = _BATCH_SIZE
+    for i in range(0, len(batch), batch_size):
+      text_batch = batch[i:i + batch_size]
+      text_batch = [
+          TextEmbeddingInput(
+              text=text, title=self.title, task_type=self.task_type)
+          for text in text_batch
+      ]
+      embeddings_batch = model.get_embeddings(text_batch)
+      embeddings.extend([el.values for el in embeddings_batch])
+    return embeddings
+
+  def load_model(self):
+    model = TextEmbeddingModel.from_pretrained(self.model_name)
+    return model
+
+
+class VertexAITextEmbeddings(EmbeddingsManager):
+  def __init__(
+      self,
+      model_name: str,
+      columns: List[str],
+      title: Optional[str] = None,
+      task_type: str = DEFAULT_TASK_TYPE,
+      project: Optional[str] = None,
+      location: Optional[str] = None,
+      credentials: Optional[Credentials] = None,
+      **kwargs):
+    """
+    Embedding Config for Vertex AI Text Embedding models following
+    https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings # pylint: disable=line-too-long
+    Text Embeddings are generated for a batch of text using the Vertex AI SDK.
+    Embeddings are returned in a list for each text in the batch. Look at
+    https://cloud.google.com/vertex-ai/docs/generative-ai/learn/model-versioning#stable-versions-available.md # pylint: disable=line-too-long
+    for more information on model versions and lifecycle.
+
+    Args:
+      model_name: The name of the Vertex AI Text Embedding model.
+      columns: The columns containing the text to be embedded.
+      task_type: The downstream task for the embeddings. Valid values are
+        RETRIEVAL_QUERY, RETRIEVAL_DOCUMENT, SEMANTIC_SIMILARITY,
+        CLASSIFICATION, CLUSTERING. For more information on the task type,
+        look at https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings # pylint: disable=line-too-long
+      title: Identifier of the text content.
+      project: The default GCP project for API calls.
+      location: The default location for API calls.
+      credentials: Custom credentials for API calls.
+        Defaults to environment credentials.
+    """
+    self.model_name = model_name
+    self.project = project
+    self.location = location
+    self.credentials = credentials
+    self.title = title
+    self.task_type = task_type
+    super().__init__(columns=columns, **kwargs)
+
+  def get_model_handler(self) -> ModelHandler:
+    return _VertexAITextEmbeddingHandler(
+        model_name=self.model_name,
+        project=self.project,
+        location=self.location,
+        credentials=self.credentials,
+        title=self.title,
+        task_type=self.task_type,
+    )
+
+  def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
+    return (
+        RunInference(
+            model_handler=_TextEmbeddingHandler(self),
+            inference_args=self.inference_args))
diff --git a/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py
new file mode 100644
index 000000000000..04a730eaefb0
--- /dev/null
+++ b/sdks/python/apache_beam/ml/transforms/embeddings/vertex_ai_test.py
@@ -0,0 +1,249 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+import uuid
+
+import apache_beam as beam
+from apache_beam.ml.inference.base import RunInference
+from apache_beam.ml.transforms import base
+from apache_beam.ml.transforms.base import MLTransform
+
+try:
+  from apache_beam.ml.transforms.embeddings.vertex_ai import VertexAITextEmbeddings
+except ImportError:
+  VertexAITextEmbeddings = None  # type: ignore
+
+# pylint: disable=ungrouped-imports
+try:
+  import tensorflow_transform as tft
+  from apache_beam.ml.transforms.tft import ScaleTo01
+except ImportError:
+  tft = None
+
+test_query = "This is a test"
+test_query_column = "feature_1"
+model_name: str = "textembedding-gecko@002"
+
+
+@unittest.skipIf(
+    VertexAITextEmbeddings is None, 'Vertex AI Python SDK is not installed.')
+class VertexAIEmbeddingsTest(unittest.TestCase):
+  def setUp(self) -> None:
+    self.artifact_location = tempfile.mkdtemp(prefix='_vertex_ai_test')
+    self.gcs_artifact_location = os.path.join(
+        'gs://temp-storage-for-perf-tests/vertex_ai', uuid.uuid4().hex)
+
+  def tearDown(self) -> None:
+    shutil.rmtree(self.artifact_location)
+
+  def test_vertex_ai_text_embeddings(self):
+    embedding_config = VertexAITextEmbeddings(
+        model_name=model_name, columns=[test_query_column])
+    with beam.Pipeline() as pipeline:
+      transformed_pcoll = (
+          pipeline
+          | "CreateData" >> beam.Create([{
+              test_query_column: test_query
+          }])
+          | "MLTransform" >> MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  embedding_config))
+
+      def assert_element(element):
+        assert len(element[test_query_column]) == 768
+
+      _ = (transformed_pcoll | beam.Map(assert_element))
+
+  @unittest.skipIf(tft is None, 'Tensorflow Transform is not installed.')
+  def test_embeddings_with_scale_to_0_1(self):
+    embedding_config = VertexAITextEmbeddings(
+        model_name=model_name,
+        columns=[test_query_column],
+    )
+    with beam.Pipeline() as pipeline:
+      transformed_pcoll = (
+          pipeline
+          | "CreateData" >> beam.Create([{
+              test_query_column: test_query
+          }])
+          | "MLTransform" >> MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  embedding_config).with_transform(
+                      ScaleTo01(columns=[test_query_column])))
+
+      def assert_element(element):
+        assert max(element.feature_1) == 1
+
+      _ = (transformed_pcoll | beam.Map(assert_element))
+
+  def pipeline_with_configurable_artifact_location(
+      self,
+      pipeline,
+      embedding_config=None,
+      read_artifact_location=None,
+      write_artifact_location=None):
+    if write_artifact_location:
+      return (
+          pipeline
+          | MLTransform(write_artifact_location=write_artifact_location).
+          with_transform(embedding_config))
+    elif read_artifact_location:
+      return (
+          pipeline
+          | MLTransform(read_artifact_location=read_artifact_location))
+    else:
+      raise NotImplementedError
+
+  def test_embeddings_with_read_artifact_location(self):
+    with beam.Pipeline() as p:
+      embedding_config = VertexAITextEmbeddings(
+          model_name=model_name, columns=[test_query_column])
+
+      with beam.Pipeline() as p:
+        data = (
+            p
+            | "CreateData" >> beam.Create([{
+                test_query_column: test_query
+            }]))
+        _ = self.pipeline_with_configurable_artifact_location(
+            pipeline=data,
+            embedding_config=embedding_config,
+            write_artifact_location=self.artifact_location)
+
+      with beam.Pipeline() as p:
+        data = (
+            p
+            | "CreateData" >> beam.Create([{
+                test_query_column: test_query
+            }, {
+                test_query_column: test_query
+            }]))
+        result_pcoll = self.pipeline_with_configurable_artifact_location(
+            pipeline=data, read_artifact_location=self.artifact_location)
+
+        def assert_element(element):
+          assert round(element, 2) == 0.15
+
+        _ = (
+            result_pcoll
+            | beam.Map(lambda x: max(x[test_query_column]))
+            #  0.14797046780586243
+            | beam.Map(assert_element))
+
+  def test_with_int_data_types(self):
+    embedding_config = VertexAITextEmbeddings(
+        model_name=model_name, columns=[test_query_column])
+    with self.assertRaises(TypeError):
+      with beam.Pipeline() as pipeline:
+        _ = (
+            pipeline
+            | "CreateData" >> beam.Create([{
+                test_query_column: 1
+            }])
+            | "MLTransform" >> MLTransform(
+                write_artifact_location=self.artifact_location).with_transform(
+                    embedding_config))
+
+  def test_with_gcs_artifact_location(self):
+    with beam.Pipeline() as p:
+      embedding_config = VertexAITextEmbeddings(
+          model_name=model_name, columns=[test_query_column])
+
+      with beam.Pipeline() as p:
+        data = (
+            p
+            | "CreateData" >> beam.Create([{
+                test_query_column: test_query
+            }]))
+        _ = self.pipeline_with_configurable_artifact_location(
+            pipeline=data,
+            embedding_config=embedding_config,
+            write_artifact_location=self.gcs_artifact_location)
+
+      with beam.Pipeline() as p:
+        data = (
+            p
+            | "CreateData" >> beam.Create([{
+                test_query_column: test_query
+            }, {
+                test_query_column: test_query
+            }]))
+        result_pcoll = self.pipeline_with_configurable_artifact_location(
+            pipeline=data, read_artifact_location=self.gcs_artifact_location)
+
+        def assert_element(element):
+          assert round(element, 2) == 0.15
+
+        _ = (
+            result_pcoll
+            | beam.Map(lambda x: max(x[test_query_column]))
+            #  0.14797046780586243
+            | beam.Map(assert_element))
+
+  def test_mltransform_to_ptransform_with_vertex(self):
+    model_name = 'textembedding-gecko@002'
+    transforms = [
+        VertexAITextEmbeddings(
+            columns=['x'],
+            model_name=model_name,
+            task_type='RETRIEVAL_DOCUMENT'),
+        VertexAITextEmbeddings(
+            columns=['y', 'z'], model_name=model_name, task_type='CLUSTERING')
+    ]
+    ptransform_mapper = base._MLTransformToPTransformMapper(
+        transforms=transforms,
+        artifact_location=self.artifact_location,
+        artifact_mode=None)
+
+    ptransform_list = ptransform_mapper.create_and_save_ptransform_list()
+    self.assertTrue(len(ptransform_list) == 2)
+
+    self.assertEqual(type(ptransform_list[0]), RunInference)
+    expected_columns = [['x'], ['y', 'z']]
+    expected_task_type = ['RETRIEVAL_DOCUMENT', 'CLUSTERING']
+    for i in range(len(ptransform_list)):
+      self.assertEqual(type(ptransform_list[i]), RunInference)
+      self.assertEqual(
+          type(ptransform_list[i]._model_handler), base._TextEmbeddingHandler)
+      self.assertEqual(
+          ptransform_list[i]._model_handler.columns, expected_columns[i])
+      self.assertEqual(
+          ptransform_list[i]._model_handler._underlying.task_type,
+          expected_task_type[i])
+      self.assertEqual(
+          ptransform_list[i]._model_handler._underlying.model_name, model_name)
+    ptransform_list = (
+        base._MLTransformToPTransformMapper.
+        load_transforms_from_artifact_location(self.artifact_location))
+    for i in range(len(ptransform_list)):
+      self.assertEqual(type(ptransform_list[i]), RunInference)
+      self.assertEqual(
+          type(ptransform_list[i]._model_handler), base._TextEmbeddingHandler)
+      self.assertEqual(
+          ptransform_list[i]._model_handler.columns, expected_columns[i])
+      self.assertEqual(
+          ptransform_list[i]._model_handler._underlying.task_type,
+          expected_task_type[i])
+      self.assertEqual(
+          ptransform_list[i]._model_handler._underlying.model_name, model_name)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/sdks/python/apache_beam/ml/transforms/handlers.py b/sdks/python/apache_beam/ml/transforms/handlers.py
index 8695d5146efa..db6ca849a625 100644
--- a/sdks/python/apache_beam/ml/transforms/handlers.py
+++ b/sdks/python/apache_beam/ml/transforms/handlers.py
@@ -17,9 +17,9 @@
 # pytype: skip-file
 
 import collections
-import hashlib
 import os
 import typing
+import uuid
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -49,6 +49,8 @@
     'TFTProcessHandler',
 ]
 
+_ID_COLUMN = 'tmp_uuid'  # Name for a temporary column.
+
 RAW_DATA_METADATA_DIR = 'raw_data_metadata'
 SCHEMA_FILE = 'schema.pbtxt'
 # tensorflow transform doesn't support the types other than tf.int64,
@@ -80,12 +82,12 @@
 tft_process_handler_output_type = typing.Union[beam.Row, Dict[str, np.ndarray]]
 
 
-class ConvertScalarValuesToListValues(beam.DoFn):
+class _ConvertScalarValuesToListValues(beam.DoFn):
   def process(
       self,
       element,
   ):
-    hash_key, element = element
+    id, element = element
     new_dict = {}
     for key, value in element.items():
       if isinstance(value,
@@ -93,10 +95,10 @@ def process(
         new_dict[key] = [value]
       else:
         new_dict[key] = value
-    yield (hash_key, new_dict)
+    yield (id, new_dict)
 
 
-class ConvertNamedTupleToDict(
+class _ConvertNamedTupleToDict(
     beam.PTransform[beam.PCollection[typing.Union[beam.Row, typing.NamedTuple]],
                     beam.PCollection[Dict[str,
                                           common_types.InstanceDictType]]]):
@@ -121,76 +123,75 @@ def expand(
       return pcoll | beam.Map(lambda x: x._asdict())
 
 
-class ComputeAndAttachHashKey(beam.DoFn):
+class _ComputeAndAttachUniqueID(beam.DoFn):
   """
-  Computes and attaches a hash key to the element.
-  Only for internal use. No backwards compatibility guarantees.
+  Computes and attaches a unique id to each element in the PCollection.
   """
   def process(self, element):
-    hash_object = hashlib.sha256()
-    for _, value in element.items():
-      # handle the case where value is a list or numpy array
-      if isinstance(value, (list, np.ndarray)):
-        hash_object.update(str(list(value)).encode())
-      else:  # assume value is a primitive that can be turned into str
-        hash_object.update(str(value).encode())
-    yield (hash_object.hexdigest(), element)
+    # UUID1 includes machine-specific bits and has a counter. As long as not too
+    # many are generated at the same time, they should be unique.
+    # UUID4 generation should be unique in practice as long as underlying random
+    # number generation is not compromised.
+    # A combintation of both should avoid the anecdotal pitfalls where
+    # replacing one with the other has helped some users.
+    # UUID collision will result in data loss, but we can detect that and fail.
+
+    # TODO(https://github.com/apache/beam/issues/29593): Evaluate MLTransform
+    # implementation without CoGBK.
+    unique_key = uuid.uuid1().bytes + uuid.uuid4().bytes
+    yield (unique_key, element)
 
 
-class GetMissingColumnsPColl(beam.DoFn):
+class _GetMissingColumns(beam.DoFn):
   """
   Returns data containing only the columns that are not
   present in the schema. This is needed since TFT only outputs
   columns that are transformed by any of the data processing transforms.
-
-  Only for internal use. No backwards compatibility guarantees.
   """
   def __init__(self, existing_columns):
     self.existing_columns = existing_columns
 
   def process(self, element):
-    new_dict = {}
-    hash_key, element = element
-    for key, value in element.items():
-      if key not in self.existing_columns:
-        new_dict[key] = value
-    yield (hash_key, new_dict)
+    id, row_dict = element
+    new_dict = {
+        k: v
+        for k, v in row_dict.items() if k not in self.existing_columns
+    }
+    yield (id, new_dict)
 
 
-class MakeHashKeyAsColumn(beam.DoFn):
+class _MakeIdAsColumn(beam.DoFn):
   """
-  Extracts the hash key from the element and adds it as a column.
-
-  Only for internal use. No backwards compatibility guarantees.
+  Extracts the id from the element and adds it as a column instead.
   """
   def process(self, element):
-    hash_key, element = element
-    element['hash_key'] = hash_key
+    id, element = element
+    element[_ID_COLUMN] = id
     yield element
 
 
-class ExtractHashAndKeyPColl(beam.DoFn):
+class _ExtractIdAndKeyPColl(beam.DoFn):
   """
-  Extracts the hash key and return hashkey and element as a tuple.
-
-  Only for internal use. No backwards compatibility guarantees.
+  Extracts the id and return id and element as a tuple.
   """
   def process(self, element):
-    hashkey = element['hash_key'][0]
-    del element['hash_key']
-    yield (hashkey.decode('utf-8'), element)
+    id = element[_ID_COLUMN][0]
+    del element[_ID_COLUMN]
+    yield (id, element)
 
 
-class MergeDicts(beam.DoFn):
+class _MergeDicts(beam.DoFn):
   """
-  Merges the dictionaries in the PCollection.
-
-  Only for internal use. No backwards compatibility guarantees.
+  Merges processed and unprocessed columns from CoGBK result into a single row.
   """
   def process(self, element):
-    _, element = element
+    unused_row_id, row_dicts_tuple = element
     new_dict = {}
-    for d in element:
+    for d in row_dicts_tuple:
+      # After CoGBK, dicts with processed and unprocessed portions of each row
+      # are wrapped in 1-element lists, since all rows have a unique id.
+      # Assertion could fail due to UUID collision.
+      assert len(d) == 1, f"Expected 1 element, got: {len(d)}."
       new_dict.update(d[0])
     yield new_dict
 
@@ -323,7 +324,7 @@ def _get_raw_data_feature_spec_per_column(
   def get_raw_data_metadata(
       self, input_types: Dict[str, type]) -> dataset_metadata.DatasetMetadata:
     raw_data_feature_spec = self.get_raw_data_feature_spec(input_types)
-    raw_data_feature_spec['hash_key'] = tf.io.VarLenFeature(dtype=tf.string)
+    raw_data_feature_spec[_ID_COLUMN] = tf.io.VarLenFeature(dtype=tf.string)
     return self.convert_raw_data_feature_spec_to_dataset_metadata(
         raw_data_feature_spec)
 
@@ -387,7 +388,7 @@ def _get_transformed_data_schema(
         transformed_types[name] = typing.Sequence[bytes]  # type: ignore[assignment]
     return transformed_types
 
-  def process_data(
+  def expand(
       self, raw_data: beam.PCollection[tft_process_handler_input_type]
   ) -> beam.PCollection[tft_process_handler_output_type]:
     """
@@ -417,14 +418,14 @@ def process_data(
         # convert Row or NamedTuple to Dict
         raw_data = (
             raw_data
-            | ConvertNamedTupleToDict().with_output_types(
+            | _ConvertNamedTupleToDict().with_output_types(
                 Dict[str, typing.Union[tuple(column_type_mapping.values())]]))  # type: ignore
         # AnalyzeAndTransformDataset raise type hint since this is
         # schema'd PCollection and the current output type would be a
         # custom type(NamedTuple) or a beam.Row type.
       else:
         column_type_mapping = self._map_column_names_to_types_from_transforms()
-        # Add hash key so TFT can output hash_key as output but as a no-op.
+        # Add id so TFT can output id as output but as a no-op.
       raw_data_metadata = self.get_raw_data_metadata(
           input_types=column_type_mapping)
       # Write untransformed metadata to a file so that it can be re-used
@@ -445,21 +446,21 @@ def process_data(
       raw_data_metadata = metadata_io.read_metadata(
           os.path.join(self.artifact_location, RAW_DATA_METADATA_DIR))
 
-    keyed_raw_data = (raw_data | beam.ParDo(ComputeAndAttachHashKey()))
+    keyed_raw_data = (raw_data | beam.ParDo(_ComputeAndAttachUniqueID()))
 
     feature_set = [feature.name for feature in raw_data_metadata.schema.feature]
-    columns_not_in_schema_with_hash = (
+    keyed_columns_not_in_schema = (
         keyed_raw_data
-        | beam.ParDo(GetMissingColumnsPColl(feature_set)))
+        | beam.ParDo(_GetMissingColumns(feature_set)))
 
     # To maintain consistency by outputting numpy array all the time,
     # whether a scalar value or list or np array is passed as input,
     #  we will convert scalar values to list values and TFT will ouput
     # numpy array all the time.
     keyed_raw_data = keyed_raw_data | beam.ParDo(
-        ConvertScalarValuesToListValues())
+        _ConvertScalarValuesToListValues())
 
-    raw_data_list = (keyed_raw_data | beam.ParDo(MakeHashKeyAsColumn()))
+    raw_data_list = (keyed_raw_data | beam.ParDo(_MakeIdAsColumn()))
 
     with tft_beam.Context(temp_dir=self.artifact_location):
       data = (raw_data_list, raw_data_metadata)
@@ -467,7 +468,7 @@ def process_data(
         transform_fn = (
             data
             | "AnalyzeDataset" >> tft_beam.AnalyzeDataset(self.process_data_fn))
-        # TODO: Remove the 'hash_key' column from the transformed
+        # TODO: Remove the 'id' column from the transformed
         # dataset schema generated by TFT.
         self.write_transform_artifacts(transform_fn, self.artifact_location)
       else:
@@ -490,7 +491,7 @@ def process_data(
       # So we will use a RowTypeConstraint to create a schema'd PCollection.
       # this is needed since new columns are included in the
       # transformed_dataset.
-      del self.transformed_schema['hash_key']
+      del self.transformed_schema[_ID_COLUMN]
       row_type = RowTypeConstraint.from_fields(
           list(self.transformed_schema.items()))
 
@@ -498,21 +499,21 @@ def process_data(
       # is not transformed by any of the transforms, then the output will
       # not have that column. So we will join the missing columns from the
       # raw_data to the transformed_dataset.
-      transformed_dataset = (
-          transformed_dataset | beam.ParDo(ExtractHashAndKeyPColl()))
+      keyed_transformed_dataset = (
+          transformed_dataset | beam.ParDo(_ExtractIdAndKeyPColl()))
 
       # The grouping is needed here since tensorflow transform only outputs
       # columns that are transformed by any of the transforms. So we will
       # join the missing columns from the raw_data to the transformed_dataset
-      # using the hash key.
+      # using the id.
       transformed_dataset = (
-          (transformed_dataset, columns_not_in_schema_with_hash)
+          (keyed_transformed_dataset, keyed_columns_not_in_schema)
           | beam.CoGroupByKey()
-          | beam.ParDo(MergeDicts()))
+          | beam.ParDo(_MergeDicts()))
 
       # The schema only contains the columns that are transformed.
       transformed_dataset = (
-          transformed_dataset | "ConvertToRowType" >>
+          transformed_dataset
+          | "ConvertToRowType" >>
           beam.Map(lambda x: beam.Row(**x)).with_output_types(row_type))
-
       return transformed_dataset
diff --git a/sdks/python/apache_beam/ml/transforms/handlers_test.py b/sdks/python/apache_beam/ml/transforms/handlers_test.py
index 327c8c76c0e9..f13a916824c4 100644
--- a/sdks/python/apache_beam/ml/transforms/handlers_test.py
+++ b/sdks/python/apache_beam/ml/transforms/handlers_test.py
@@ -298,7 +298,7 @@ def test_tft_process_handler_verify_artifacts(self):
           transforms=[tft.ScaleTo01(columns=['x'])],
           artifact_location=self.artifact_location,
       )
-      _ = process_handler.process_data(raw_data)
+      _ = raw_data | process_handler
 
       self.assertTrue(
           os.path.exists(
@@ -315,7 +315,7 @@ def test_tft_process_handler_verify_artifacts(self):
       raw_data = (p | beam.Create([{'x': np.array([2, 5])}]))
       process_handler = handlers.TFTProcessHandler(
           artifact_location=self.artifact_location, artifact_mode='consume')
-      transformed_data = process_handler.process_data(raw_data)
+      transformed_data = raw_data | process_handler
       transformed_data |= beam.Map(lambda x: x.x)
 
       # the previous min is 1 and max is 6. So this should scale by (1, 6)
@@ -494,7 +494,7 @@ def test_tft_process_handler_unused_column(self):
           transforms=[scale_to_0_1_fn],
           artifact_location=self.artifact_location,
       )
-      transformed_pcoll = process_handler.process_data(raw_data)
+      transformed_pcoll = raw_data | process_handler
       transformed_pcoll_x = transformed_pcoll | beam.Map(lambda x: x.x)
       transformed_pcoll_y = transformed_pcoll | beam.Map(lambda x: x.y)
       assert_that(
@@ -520,7 +520,7 @@ def test_consume_mode_with_extra_columns_in_the_input(self):
           transforms=[tft.ScaleTo01(columns=['x'])],
           artifact_location=self.artifact_location,
       )
-      _ = process_handler.process_data(raw_data)
+      _ = raw_data | process_handler
 
     test_data = [{
         'x': np.array([2, 5]), 'y': np.array([1, 2]), 'z': 'fake_string'
@@ -548,7 +548,7 @@ def test_consume_mode_with_extra_columns_in_the_input(self):
       raw_data = (p | beam.Create(test_data))
       process_handler = handlers.TFTProcessHandler(
           artifact_location=self.artifact_location, artifact_mode='consume')
-      transformed_data = process_handler.process_data(raw_data)
+      transformed_data = raw_data | process_handler
 
       transformed_data_x = transformed_data | beam.Map(lambda x: x.x)
       transformed_data_y = transformed_data | beam.Map(lambda x: x.y)
@@ -569,6 +569,52 @@ def test_consume_mode_with_extra_columns_in_the_input(self):
           equal_to(expected_test_data_z, equals_fn=np.array_equal),
           label='unused column: z')
 
+  def test_handler_with_same_input_elements(self):
+    with beam.Pipeline() as p:
+      data = [
+          {
+              'x': 'I'
+          },
+          {
+              'x': 'love'
+          },
+          {
+              'x': 'Beam'
+          },
+          {
+              'x': 'Beam'
+          },
+          {
+              'x': 'is'
+          },
+          {
+              'x': 'awesome'
+          },
+      ]
+      raw_data = (p | beam.Create(data))
+      process_handler = handlers.TFTProcessHandler(
+          transforms=[tft.ComputeAndApplyVocabulary(columns=['x'])],
+          artifact_location=self.artifact_location,
+      )
+      transformed_data = raw_data | process_handler
+
+      expected_data = [
+          beam.Row(x=np.array([4])),
+          beam.Row(x=np.array([1])),
+          beam.Row(x=np.array([0])),
+          beam.Row(x=np.array([0])),
+          beam.Row(x=np.array([2])),
+          beam.Row(x=np.array([3])),
+      ]
+
+      expected_data_x = [row.x for row in expected_data]
+      actual_data_x = transformed_data | beam.Map(lambda x: x.x)
+
+      assert_that(
+          actual_data_x,
+          equal_to(expected_data_x, equals_fn=np.array_equal),
+          label='transformed data')
+
 
 if __name__ == '__main__':
   unittest.main()
diff --git a/sdks/python/apache_beam/ml/transforms/tft.py b/sdks/python/apache_beam/ml/transforms/tft.py
index c7b8ff015324..f1ff0e4dfe57 100644
--- a/sdks/python/apache_beam/ml/transforms/tft.py
+++ b/sdks/python/apache_beam/ml/transforms/tft.py
@@ -42,6 +42,7 @@
 from typing import Tuple
 from typing import Union
 
+import apache_beam as beam
 import tensorflow as tf
 import tensorflow_transform as tft
 from apache_beam.ml.transforms.base import BaseOperation
@@ -95,6 +96,20 @@ def __init__(self, columns: List[str]) -> None:
           "Columns are not specified. Please specify the column for the "
           " op %s" % self.__class__.__name__)
 
+  def get_ptransform_for_processing(self, **kwargs) -> beam.PTransform:
+    from apache_beam.ml.transforms.handlers import TFTProcessHandler
+    params = {}
+    artifact_location = kwargs.get('artifact_location')
+    if not artifact_location:
+      raise RuntimeError(
+          "artifact_location is not specified. Please specify the "
+          "artifact_location for the op %s" % self.__class__.__name__)
+
+    artifact_mode = kwargs.get('artifact_mode')
+    if artifact_mode:
+      params['artifact_mode'] = artifact_mode
+    return TFTProcessHandler(artifact_location=artifact_location, **params)
+
   @tf.function
   def _split_string_with_delimiter(self, data, delimiter):
     """
@@ -157,8 +172,8 @@ def __init__(
       num_oov_buckets:  Any lookup of an out-of-vocabulary token will return a
         bucket ID based on its hash if `num_oov_buckets` is greater than zero.
         Otherwise it is assigned the `default_value`.
-      vocab_filename: The file name for the vocabulary file. If not provided,
-        the default name would be `compute_and_apply_vocab'
+      vocab_filename: The file name for the vocabulary file. The vocab file
+        will be suffixed with the column name.
         NOTE in order to make your pipelines resilient to implementation
         details please set `vocab_filename` when you are using
         the vocab_filename on a downstream component.
@@ -168,8 +183,7 @@ def __init__(
     self._top_k = top_k
     self._frequency_threshold = frequency_threshold
     self._num_oov_buckets = num_oov_buckets
-    self._vocab_filename = vocab_filename if vocab_filename else (
-        'compute_and_apply_vocab')
+    self._vocab_filename = vocab_filename
     self._name = name
     self.split_string_by_delimiter = split_string_by_delimiter
 
@@ -181,6 +195,9 @@ def apply_transform(
       data = self._split_string_with_delimiter(
           data, self.split_string_by_delimiter)
 
+    vocab_filename = self._vocab_filename
+    if vocab_filename:
+      vocab_filename = vocab_filename + f'_{output_column_name}'
     return {
         output_column_name: tft.compute_and_apply_vocabulary(
             x=data,
@@ -188,7 +205,7 @@ def apply_transform(
             top_k=self._top_k,
             frequency_threshold=self._frequency_threshold,
             num_oov_buckets=self._num_oov_buckets,
-            vocab_filename=self._vocab_filename,
+            vocab_filename=vocab_filename,
             name=self._name)
     }
 
@@ -520,7 +537,7 @@ def __init__(
       ngram_range: Tuple[int, int] = (1, 1),
       ngrams_separator: Optional[str] = None,
       compute_word_count: bool = False,
-      key_vocab_filename: str = 'key_vocab_mapping',
+      key_vocab_filename: Optional[str] = None,
       name: Optional[str] = None,
   ):
     """
@@ -543,7 +560,9 @@ def __init__(
       compute_word_count: A boolean that specifies whether to compute
         the unique word count over the entire dataset. Defaults to False.
       key_vocab_filename: The file name for the key vocabulary file when
-        compute_word_count is True.
+        compute_word_count is True. If empty, a file name 
+        will be chosen based on the current scope. If provided, the vocab
+        file will be suffixed with the column name.
       name: A name for the operation (optional).
 
     Note that original order of the input may not be preserved.
@@ -570,10 +589,14 @@ def apply_transform(self, data: tf.SparseTensor, output_col_name: str):
           data, self.split_string_by_delimiter)
     output = tft.bag_of_words(
         data, self.ngram_range, self.ngrams_separator, self.name)
-    # word counts are written to the key_vocab_filename
-    self.compute_word_count_fn(data, self.key_vocab_filename)
+    # word counts are written to the file only if compute_word_count is True
+    key_vocab_filename = self.key_vocab_filename
+    if key_vocab_filename:
+      key_vocab_filename = key_vocab_filename + f'_{output_col_name}'
+    self.compute_word_count_fn(data, key_vocab_filename)
     return {output_col_name: output}
 
 
-def count_unqiue_words(data: tf.SparseTensor, output_vocab_name: str) -> None:
+def count_unqiue_words(
+    data: tf.SparseTensor, output_vocab_name: Optional[str]) -> None:
   tft.count_per_key(data, key_vocabulary_filename=output_vocab_name)
diff --git a/sdks/python/apache_beam/ml/transforms/tft_test.py b/sdks/python/apache_beam/ml/transforms/tft_test.py
index 38ded6a809af..558b4ede2ec6 100644
--- a/sdks/python/apache_beam/ml/transforms/tft_test.py
+++ b/sdks/python/apache_beam/ml/transforms/tft_test.py
@@ -357,6 +357,85 @@ def test_string_split_with_multiple_delimiters(self):
       ]
       assert_that(result, equal_to(expected_result, equals_fn=np.array_equal))
 
+  def test_multiple_columns_with_default_vocab_name(self):
+    data = [{
+        'x': ['I', 'like', 'pie'], 'y': ['Apach', 'Beam', 'is', 'awesome']
+    },
+            {
+                'x': ['yum', 'yum', 'pie'],
+                'y': ['Beam', 'is', 'a', 'unified', 'model']
+            }]
+    with beam.Pipeline() as p:
+      result = (
+          p
+          | "Create" >> beam.Create(data)
+          | "MLTransform" >> base.MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  tft.ComputeAndApplyVocabulary(columns=['x', 'y'])))
+
+      expected_data_x = [np.array([3, 2, 1]), np.array([0, 0, 1])]
+
+      expected_data_y = [np.array([6, 1, 0, 4]), np.array([1, 0, 5, 2, 3])]
+
+      actual_data_x = (result | beam.Map(lambda x: x.x))
+      actual_data_y = (result | beam.Map(lambda x: x.y))
+
+      assert_that(
+          actual_data_x,
+          equal_to(expected_data_x, equals_fn=np.array_equal),
+          label='x')
+      assert_that(
+          actual_data_y,
+          equal_to(expected_data_y, equals_fn=np.array_equal),
+          label='y')
+    files = os.listdir(self.artifact_location)
+    files.remove(base._ATTRIBUTE_FILE_NAME)
+    assert len(files) == 1
+    tft_vocab_assets = os.listdir(
+        os.path.join(
+            self.artifact_location, files[0], 'transform_fn', 'assets'))
+    assert len(tft_vocab_assets) == 2
+
+  def test_multiple_columns_with_vocab_name(self):
+    data = [{
+        'x': ['I', 'like', 'pie'], 'y': ['Apach', 'Beam', 'is', 'awesome']
+    },
+            {
+                'x': ['yum', 'yum', 'pie'],
+                'y': ['Beam', 'is', 'a', 'unified', 'model']
+            }]
+    with beam.Pipeline() as p:
+      result = (
+          p
+          | "Create" >> beam.Create(data)
+          | "MLTransform" >> base.MLTransform(
+              write_artifact_location=self.artifact_location).with_transform(
+                  tft.ComputeAndApplyVocabulary(
+                      columns=['x', 'y'], vocab_filename='my_vocab')))
+
+      expected_data_x = [np.array([3, 2, 1]), np.array([0, 0, 1])]
+
+      expected_data_y = [np.array([6, 1, 0, 4]), np.array([1, 0, 5, 2, 3])]
+
+      actual_data_x = (result | beam.Map(lambda x: x.x))
+      actual_data_y = (result | beam.Map(lambda x: x.y))
+
+      assert_that(
+          actual_data_x,
+          equal_to(expected_data_x, equals_fn=np.array_equal),
+          label='x')
+      assert_that(
+          actual_data_y,
+          equal_to(expected_data_y, equals_fn=np.array_equal),
+          label='y')
+    files = os.listdir(self.artifact_location)
+    files.remove(base._ATTRIBUTE_FILE_NAME)
+    assert len(files) == 1
+    tft_vocab_assets = os.listdir(
+        os.path.join(
+            self.artifact_location, files[0], 'transform_fn', 'assets'))
+    assert len(tft_vocab_assets) == 2
+
 
 class TFIDIFTest(unittest.TestCase):
   def setUp(self) -> None:
@@ -711,8 +790,13 @@ def test_count_per_key_on_list(self):
               ]))
 
     def validate_count_per_key(key_vocab_filename):
+      files = os.listdir(self.artifact_location)
+      files.remove(base._ATTRIBUTE_FILE_NAME)
       key_vocab_location = os.path.join(
-          self.artifact_location, 'transform_fn/assets', key_vocab_filename)
+          self.artifact_location,
+          files[0],
+          'transform_fn/assets',
+          key_vocab_filename + '_x')
       with open(key_vocab_location, 'r') as f:
         key_vocab_list = [line.strip() for line in f]
       return key_vocab_list
diff --git a/sdks/python/apache_beam/ml/transforms/utils.py b/sdks/python/apache_beam/ml/transforms/utils.py
index 19bb02c5ae1b..fadf611b0e66 100644
--- a/sdks/python/apache_beam/ml/transforms/utils.py
+++ b/sdks/python/apache_beam/ml/transforms/utils.py
@@ -17,9 +17,11 @@
 
 __all__ = ['ArtifactsFetcher']
 
+import os
 import typing
 
 import tensorflow_transform as tft
+from apache_beam.ml.transforms import base
 
 
 class ArtifactsFetcher():
@@ -28,8 +30,18 @@ class ArtifactsFetcher():
   to the TFTProcessHandlers in MLTransform.
   """
   def __init__(self, artifact_location):
-    self.artifact_location = artifact_location
-    self.transform_output = tft.TFTransformOutput(self.artifact_location)
+    files = os.listdir(artifact_location)
+    files.remove(base._ATTRIBUTE_FILE_NAME)
+    # TODO: https://github.com/apache/beam/issues/29356
+    #  Integrate ArtifactFetcher into MLTransform.
+    if len(files) > 1:
+      raise NotImplementedError(
+          "MLTransform may have been utilized alongside transforms written "
+          "in TensorFlow Transform, in conjunction with those from different "
+          "frameworks. Currently, retrieving artifacts from this "
+          "multi-framework setup is not supported.")
+    self._artifact_location = os.path.join(artifact_location, files[0])
+    self.transform_output = tft.TFTransformOutput(self._artifact_location)
 
   def get_vocab_list(
       self,
diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py
index 2dba78948444..030a25aa85c2 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -36,6 +36,7 @@
 from apache_beam.options.value_provider import StaticValueProvider
 from apache_beam.options.value_provider import ValueProvider
 from apache_beam.transforms.display import HasDisplayData
+from apache_beam.utils import proto_utils
 
 __all__ = [
     'PipelineOptions',
@@ -390,6 +391,36 @@ def get_all_options(
 
     return result
 
+  def to_runner_api(self):
+    def to_struct_value(o):
+      if isinstance(o, (bool, int, str)):
+        return o
+      elif isinstance(o, (tuple, list)):
+        return [to_struct_value(e) for e in o]
+      elif isinstance(o, dict):
+        return {str(k): to_struct_value(v) for k, v in o.items()}
+      else:
+        return str(o)  # Best effort.
+
+    return proto_utils.pack_Struct(
+        **{
+            f'beam:option:{k}:v1': to_struct_value(v)
+            for (k, v) in self.get_all_options(
+                drop_default=True, retain_unknown_options=True).items()
+            if v is not None
+        })
+
+  @classmethod
+  def from_runner_api(cls, proto_options):
+    def from_urn(key):
+      assert key.startswith('beam:option:')
+      assert key.endswith(':v1')
+      return key[12:-3]
+
+    return cls(
+        **{from_urn(key): value
+           for (key, value) in proto_options.items()})
+
   def display_data(self):
     return self.get_all_options(drop_default=True, retain_unknown_options=True)
 
@@ -528,13 +559,31 @@ def _add_argparse_args(cls, parser):
         'exception if a transform is created with a non-unique label.')
 
 
+class StreamingOptions(PipelineOptions):
+  @classmethod
+  def _add_argparse_args(cls, parser):
+    parser.add_argument(
+        '--update_compatibility_version',
+        default=None,
+        help='Attempt to produce a pipeline compatible with the given prior '
+        'version of the Beam SDK. '
+        'See for example, https://cloud.google.com/dataflow/docs/guides/'
+        'updating-a-pipeline')
+
+
 class CrossLanguageOptions(PipelineOptions):
+  @staticmethod
+  def _beam_services_from_enviroment():
+    return json.loads(os.environ.get('BEAM_SERVICE_OVERRIDES') or '{}')
+
   @classmethod
   def _add_argparse_args(cls, parser):
     parser.add_argument(
         '--beam_services',
-        type=json.loads,
-        default={},
+        type=lambda s: {
+            **cls._beam_services_from_enviroment(), **json.loads(s)
+        },
+        default=cls._beam_services_from_enviroment(),
         help=(
             'For convenience, Beam provides the ability to automatically '
             'download and start various services (such as expansion services) '
@@ -543,7 +592,9 @@ def _add_argparse_args(cls, parser):
             'use pre-started services or non-default pre-existing artifacts to '
             'start the given service. '
             'Should be a json mapping of gradle build targets to pre-built '
-            'artifacts (e.g. jar files) expansion endpoints (e.g. host:port).'))
+            'artifacts (e.g. jar files) or expansion endpoints '
+            '(e.g. host:port). Defaults to the value of BEAM_SERVICE_OVERRIDES '
+            'from the environment.'))
 
     parser.add_argument(
         '--use_transform_service',
diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py
index 2d85d055eafb..c7a56b4739bd 100644
--- a/sdks/python/apache_beam/options/pipeline_options_test.py
+++ b/sdks/python/apache_beam/options/pipeline_options_test.py
@@ -21,11 +21,14 @@
 
 import json
 import logging
+import os
 import unittest
 
 import hamcrest as hc
+import mock
 from parameterized import parameterized
 
+from apache_beam.options.pipeline_options import CrossLanguageOptions
 from apache_beam.options.pipeline_options import DebugOptions
 from apache_beam.options.pipeline_options import GoogleCloudOptions
 from apache_beam.options.pipeline_options import PipelineOptions
@@ -395,6 +398,36 @@ def test_worker_options(self):
     self.assertEqual(worker_options.machine_type, 'abc')
     self.assertEqual(worker_options.disk_type, 'def')
 
+  def test_beam_services_empty(self):
+    with mock.patch.dict(os.environ, {}, clear=True):
+      options = PipelineOptions().view_as(CrossLanguageOptions)
+      self.assertEqual(options.beam_services, {})
+
+  def test_beam_services_from_env(self):
+    with mock.patch.dict(os.environ,
+                         {'BEAM_SERVICE_OVERRIDES': '{"foo": "bar"}'},
+                         clear=True):
+      options = PipelineOptions().view_as(CrossLanguageOptions)
+      self.assertEqual(options.beam_services, {'foo': 'bar'})
+
+  def test_beam_services_from_flag(self):
+    with mock.patch.dict(os.environ, {}, clear=True):
+      options = PipelineOptions(['--beam_services={"foo": "bar"}'
+                                 ]).view_as(CrossLanguageOptions)
+      self.assertEqual(options.beam_services, {'foo': 'bar'})
+
+  def test_beam_services_from_env_and_flag(self):
+    with mock.patch.dict(
+        os.environ,
+        {'BEAM_SERVICE_OVERRIDES': '{"foo": "bar", "other": "zzz"}'},
+        clear=True):
+      options = PipelineOptions(['--beam_services={"foo": "override"}'
+                                 ]).view_as(CrossLanguageOptions)
+      self.assertEqual(
+          options.beam_services, {
+              'foo': 'override', 'other': 'zzz'
+          })
+
   def test_option_modifications_are_shared_between_views(self):
     pipeline_options = PipelineOptions([
         '--mock_option',
diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py
index ed0736250d1f..53044982a066 100644
--- a/sdks/python/apache_beam/pipeline.py
+++ b/sdks/python/apache_beam/pipeline.py
@@ -48,6 +48,7 @@
 # mypy: disallow-untyped-defs
 
 import abc
+import contextlib
 import logging
 import os
 import re
@@ -590,9 +591,12 @@ def run(self, test_runner_api='AUTO'):
 
   def __enter__(self):
     # type: () -> Pipeline
-    self._extra_context = subprocess_server.JavaJarServer.beam_services(
-        self._options.view_as(CrossLanguageOptions).beam_services)
-    self._extra_context.__enter__()
+    self._extra_context = contextlib.ExitStack()
+    self._extra_context.enter_context(
+        subprocess_server.JavaJarServer.beam_services(
+            self._options.view_as(CrossLanguageOptions).beam_services))
+    self._extra_context.enter_context(
+        subprocess_server.SubprocessServer.cache_subprocesses())
     return self
 
   def __exit__(
diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py
index 113d1a99990c..38d8b1f03a11 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -872,7 +872,9 @@ def test_dir(self):
         'slices',
         'style',
         'view_as',
-        'display_data'
+        'display_data',
+        'from_runner_api',
+        'to_runner_api',
     },
                      {
                          attr
@@ -884,7 +886,9 @@ def test_dir(self):
         'get_all_options',
         'style',
         'view_as',
-        'display_data'
+        'display_data',
+        'from_runner_api',
+        'to_runner_api',
     },
                      {
                          attr
diff --git a/sdks/python/apache_beam/portability/common_urns.py b/sdks/python/apache_beam/portability/common_urns.py
index 3799af5d2e1b..e7b086c5a649 100644
--- a/sdks/python/apache_beam/portability/common_urns.py
+++ b/sdks/python/apache_beam/portability/common_urns.py
@@ -34,6 +34,7 @@
 StandardPTransforms = beam_runner_api_pb2_urns.StandardPTransforms
 StandardRequirements = beam_runner_api_pb2_urns.StandardRequirements
 StandardResourceHints = beam_runner_api_pb2_urns.StandardResourceHints
+StandardRunnerProtocols = beam_runner_api_pb2_urns.StandardRunnerProtocols
 StandardSideInputTypes = beam_runner_api_pb2_urns.StandardSideInputTypes
 StandardUserStateTypes = beam_runner_api_pb2_urns.StandardUserStateTypes
 ExpansionMethods = external_transforms_pb2_urns.ExpansionMethods
@@ -73,6 +74,7 @@
 monitoring_info_labels = MonitoringInfo.MonitoringInfoLabels
 
 protocols = StandardProtocols.Enum
+runner_protocols = StandardRunnerProtocols.Enum
 requirements = StandardRequirements.Enum
 
 displayData = StandardDisplayData.DisplayData
diff --git a/sdks/python/apache_beam/runners/dask/transform_evaluator.py b/sdks/python/apache_beam/runners/dask/transform_evaluator.py
index c4aac7f2111f..d4d58879b7fe 100644
--- a/sdks/python/apache_beam/runners/dask/transform_evaluator.py
+++ b/sdks/python/apache_beam/runners/dask/transform_evaluator.py
@@ -24,6 +24,7 @@
 """
 import abc
 import dataclasses
+import math
 import typing as t
 
 import apache_beam
@@ -59,7 +60,10 @@ def apply(self, input_bag: OpInput) -> db.Bag:
     assert input_bag is None, 'Create expects no input!'
     original_transform = t.cast(_Create, self.transform)
     items = original_transform.values
-    return db.from_sequence(items)
+    return db.from_sequence(
+        items,
+        partition_size=max(
+            1, math.ceil(math.sqrt(len(items)) / math.sqrt(100))))
 
 
 class ParDo(DaskBagOp):
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_job_service.py b/sdks/python/apache_beam/runners/dataflow/dataflow_job_service.py
index 5ba6dbed4e64..2bd8605c9928 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_job_service.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_job_service.py
@@ -19,6 +19,7 @@
 import logging
 import sys
 
+from apache_beam.portability.api import beam_job_api_pb2
 from apache_beam.runners.dataflow import dataflow_runner
 from apache_beam.runners.portability import local_job_service
 from apache_beam.runners.portability import local_job_service_main
@@ -34,6 +35,11 @@ def _invoke_runner(self):
     runner = dataflow_runner.DataflowRunner()
     self.result = runner.run_pipeline(
         None, self.pipeline_options(), self._pipeline_proto)
+    # The result can be None if there is no need to send a request
+    # to the service (e.g. template creation).
+    if not getattr(self.result, 'has_job', None):
+      self.set_state(beam_job_api_pb2.JobState.DONE)
+      return self.result
     # Prefer this to result.wait_until_finish() to get state updates
     # and avoid creating an extra thread (which also messes with logging).
     dataflow_runner.DataflowRunner.poll_for_job_completion(
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_job_service_test.py b/sdks/python/apache_beam/runners/dataflow/dataflow_job_service_test.py
index 896496d43775..e2f880085cb8 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_job_service_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_job_service_test.py
@@ -48,6 +48,29 @@ def test_dry_run(self):
           temp_location='gs://bucket/dir',
           region='us-central1',
           dry_run=True,
+          template_location='gs://bucket/dir',
+      )
+      with beam.Pipeline(options=options) as p:
+        _ = p | beam.Create([1, 2, 3]) | beam.Map(lambda x: x * x)
+    finally:
+      job_servicer.stop()
+
+
+@unittest.skipIf(apiclient is None, 'GCP dependencies are not installed')
+class DirectPipelineTemplateTest(unittest.TestCase):
+  def test_template(self):
+    job_servicer = local_job_service.LocalJobServicer(
+        None, beam_job_type=dataflow_job_service.DataflowBeamJob)
+    port = job_servicer.start_grpc_server(0)
+    try:
+      options = PipelineOptions(
+          runner='PortableRunner',
+          job_endpoint=f'localhost:{port}',
+          project='apache-beam-testing',
+          region='us-central1',
+          staging_location='gs://apache-beam-testing-stg/stg/',
+          temp_location='gs://apache-beam-testing-temp/tmp',
+          template_location='gs://apache-beam-testing-temp/test/template',
       )
       with beam.Pipeline(options=options) as p:
         _ = p | beam.Create([1, 2, 3]) | beam.Map(lambda x: x * x)
diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
index 7ad6ab04be68..dc315119e480 100644
--- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
+++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py
@@ -47,6 +47,7 @@
 from apache_beam.runners.runner import PipelineResult
 from apache_beam.runners.runner import PipelineRunner
 from apache_beam.runners.runner import PipelineState
+from apache_beam.transforms import environments
 from apache_beam.typehints import typehints
 from apache_beam.utils import processes
 from apache_beam.utils.interactive_utils import is_in_notebook
@@ -380,7 +381,6 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None):
       self.proto_pipeline = pipeline_proto
 
     else:
-      from apache_beam.transforms import environments
       if options.view_as(SetupOptions).prebuild_sdk_container_engine:
         # if prebuild_sdk_container_engine is specified we will build a new sdk
         # container image with dependencies pre-installed and use that image,
@@ -414,6 +414,12 @@ def run_pipeline(self, pipeline, options, pipeline_proto=None):
       self.proto_pipeline, self.proto_context = pipeline.to_runner_api(
           return_context=True, default_environment=self._default_environment)
 
+    # Dataflow can only handle Docker environments.
+    for env_id, env in self.proto_pipeline.components.environments.items():
+      self.proto_pipeline.components.environments[env_id].CopyFrom(
+          environments.resolve_anyof_environment(
+              env, common_urns.environments.DOCKER.urn))
+
     # Optimize the pipeline if it not streaming and the pre_optimize
     # experiment is set.
     if not options.view_as(StandardOptions).streaming:
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index c313268cafc5..2ef50b3c2edb 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -514,7 +514,11 @@ def __init__(self, options, root_staging_location=None):
       # client in terms of which GCP project to use.
       self._storage_client = storage.Client(
           credentials=storage_credentials,
-          project=self.google_cloud_options.project)
+          project=self.google_cloud_options.project,
+          extra_headers={
+              "User-Agent": "apache-beam/%s (GPN:Beam)" %
+              beam_version.__version__
+          })
     else:
       self._storage_client = storage.Client.create_anonymous_client()
     self._sdk_image_overrides = self._get_sdk_image_overrides(options)
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/names.py b/sdks/python/apache_beam/runners/dataflow/internal/names.py
index fa54c6ce126e..8d3dc94a3cee 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/names.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/names.py
@@ -34,6 +34,6 @@
 # Unreleased sdks use container image tag specified below.
 # Update this tag whenever there is a change that
 # requires changes to SDK harness container or SDK harness launcher.
-BEAM_DEV_SDK_CONTAINER_TAG = 'beam-master-20231116'
+BEAM_DEV_SDK_CONTAINER_TAG = 'beam-master-20231227'
 
 DATAFLOW_CONTAINER_IMAGE_REPOSITORY = 'gcr.io/cloud-dataflow/v1beta3'
diff --git a/sdks/python/apache_beam/runners/direct/direct_runner.py b/sdks/python/apache_beam/runners/direct/direct_runner.py
index db53e4122bbc..a470ba80d8ee 100644
--- a/sdks/python/apache_beam/runners/direct/direct_runner.py
+++ b/sdks/python/apache_beam/runners/direct/direct_runner.py
@@ -507,6 +507,17 @@ def run_pipeline(self, pipeline, options):
     from apache_beam.runners.direct.transform_evaluator import \
       TransformEvaluatorRegistry
     from apache_beam.testing.test_stream import TestStream
+    from apache_beam.transforms.external import ExternalTransform
+
+    class VerifyNoCrossLanguageTransforms(PipelineVisitor):
+      """Visitor determining whether a Pipeline uses a TestStream."""
+      def visit_transform(self, applied_ptransform):
+        if isinstance(applied_ptransform.transform, ExternalTransform):
+          raise RuntimeError(
+              "Streaming Python direct runner "
+              "does not support cross-language pipelines.")
+
+    pipeline.visit(VerifyNoCrossLanguageTransforms())
 
     # If the TestStream I/O is used, use a mock test clock.
     class TestStreamUsageVisitor(PipelineVisitor):
diff --git a/sdks/python/apache_beam/runners/interactive/utils.py b/sdks/python/apache_beam/runners/interactive/utils.py
index 78fc26b95e69..524e532c2e3e 100644
--- a/sdks/python/apache_beam/runners/interactive/utils.py
+++ b/sdks/python/apache_beam/runners/interactive/utils.py
@@ -30,6 +30,7 @@
 import pandas as pd
 
 import apache_beam as beam
+from apache_beam import version as beam_version
 from apache_beam.dataframe.convert import to_pcollection
 from apache_beam.dataframe.frame_base import DeferredBase
 from apache_beam.internal.gcp import auth
@@ -457,7 +458,12 @@ def assert_bucket_exists(bucket_name):
       # We set project to None, so it will not try to use project id from
       # the environment (ADC).
       storage_client = storage.Client(
-          credentials=credentials.get_google_auth_credentials(), project=None)
+          credentials=credentials.get_google_auth_credentials(),
+          project=None,
+          extra_headers={
+              "User-Agent": "apache-beam/%s (GPN:Beam)" %
+              beam_version.__version__
+          })
     else:
       storage_client = storage.Client.create_anonymous_client()
     storage_client.get_bucket(bucket_name)
diff --git a/sdks/python/apache_beam/runners/portability/expansion_service.py b/sdks/python/apache_beam/runners/portability/expansion_service.py
index 8be9d98508ed..c7728098f30c 100644
--- a/sdks/python/apache_beam/runners/portability/expansion_service.py
+++ b/sdks/python/apache_beam/runners/portability/expansion_service.py
@@ -19,9 +19,12 @@
 """
 # pytype: skip-file
 
+import copy
 import traceback
 
 from apache_beam import pipeline as beam_pipeline
+from apache_beam.options import pipeline_options
+from apache_beam.portability import common_urns
 from apache_beam.portability import python_urns
 from apache_beam.portability.api import beam_expansion_api_pb2
 from apache_beam.portability.api import beam_expansion_api_pb2_grpc
@@ -33,15 +36,33 @@
 
 class ExpansionServiceServicer(
     beam_expansion_api_pb2_grpc.ExpansionServiceServicer):
-  def __init__(self, options=None):
+  def __init__(self, options=None, loopback_address=None):
     self._options = options or beam_pipeline.PipelineOptions(
         environment_type=python_urns.EMBEDDED_PYTHON, sdk_location='container')
-    self._default_environment = (
-        environments.Environment.from_options(self._options))
+    default_environment = (environments.Environment.from_options(self._options))
+    if loopback_address:
+      loopback_environment = environments.Environment.from_options(
+          beam_pipeline.PipelineOptions(
+              environment_type=common_urns.environments.EXTERNAL.urn,
+              environment_config=loopback_address))
+      default_environment = environments.AnyOfEnvironment(
+          [default_environment, loopback_environment])
+    self._default_environment = default_environment
 
   def Expand(self, request, context=None):
     try:
-      pipeline = beam_pipeline.Pipeline(options=self._options)
+      options = copy.deepcopy(self._options)
+      request_options = pipeline_options.PipelineOptions.from_runner_api(
+          request.pipeline_options)
+      # TODO(https://github.com/apache/beam/issues/20090): Figure out the
+      # correct subset of options to apply to expansion.
+      if request_options.view_as(
+          pipeline_options.StreamingOptions).update_compatibility_version:
+        options.view_as(
+            pipeline_options.StreamingOptions
+        ).update_compatibility_version = request_options.view_as(
+            pipeline_options.StreamingOptions).update_compatibility_version
+      pipeline = beam_pipeline.Pipeline(options=options)
 
       def with_pipeline(component, pcoll_id=None):
         component.pipeline = pipeline
diff --git a/sdks/python/apache_beam/runners/portability/expansion_service_main.py b/sdks/python/apache_beam/runners/portability/expansion_service_main.py
index 30cbdccb596e..307f6bd54182 100644
--- a/sdks/python/apache_beam/runners/portability/expansion_service_main.py
+++ b/sdks/python/apache_beam/runners/portability/expansion_service_main.py
@@ -28,8 +28,10 @@
 from apache_beam.options.pipeline_options import SetupOptions
 from apache_beam.portability.api import beam_artifact_api_pb2_grpc
 from apache_beam.portability.api import beam_expansion_api_pb2_grpc
+from apache_beam.portability.api import beam_fn_api_pb2_grpc
 from apache_beam.runners.portability import artifact_service
 from apache_beam.runners.portability import expansion_service
+from apache_beam.runners.worker import worker_pool_main
 from apache_beam.transforms import fully_qualified_named_transform
 from apache_beam.utils import thread_pool_executor
 
@@ -41,6 +43,7 @@ def main(argv):
   parser.add_argument(
       '-p', '--port', type=int, help='port on which to serve the job api')
   parser.add_argument('--fully_qualified_name_glob', default=None)
+  parser.add_argument('--serve_loopback_worker', action='store_true')
   known_args, pipeline_args = parser.parse_known_args(argv)
   pipeline_options = PipelineOptions(
       pipeline_args + ["--experiments=beam_fn_api", "--sdk_location=container"])
@@ -52,14 +55,23 @@ def main(argv):
   with fully_qualified_named_transform.FullyQualifiedNamedTransform.with_filter(
       known_args.fully_qualified_name_glob):
 
+    address = '[::]:{}'.format(known_args.port)
     server = grpc.server(thread_pool_executor.shared_unbounded_instance())
+    if known_args.serve_loopback_worker:
+      beam_fn_api_pb2_grpc.add_BeamFnExternalWorkerPoolServicer_to_server(
+          worker_pool_main.BeamFnExternalWorkerPoolServicer(), server)
+      loopback_address = address
+    else:
+      loopback_address = None
     beam_expansion_api_pb2_grpc.add_ExpansionServiceServicer_to_server(
-        expansion_service.ExpansionServiceServicer(pipeline_options), server)
+        expansion_service.ExpansionServiceServicer(
+            pipeline_options, loopback_address=loopback_address),
+        server)
     beam_artifact_api_pb2_grpc.add_ArtifactRetrievalServiceServicer_to_server(
         artifact_service.ArtifactRetrievalService(
             artifact_service.BeamFilesystemHandler(None).file_reader),
         server)
-    server.add_insecure_port('[::]:{}'.format(known_args.port))
+    server.add_insecure_port(address)
     server.start()
     _LOGGER.info('Listening for expansion requests at %d', known_args.port)
 
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py
index bc60b5dd86cd..885c96146456 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/execution.py
@@ -23,6 +23,7 @@
 import copy
 import itertools
 import logging
+import struct
 import typing
 import uuid
 import weakref
@@ -351,7 +352,7 @@ def append(self, elements_data):
         self._values_by_window[key, window].append(value)
 
   def encoded_items(self):
-    # type: () -> Iterator[Tuple[bytes, bytes, bytes]]
+    # type: () -> Iterator[Tuple[bytes, bytes, bytes, int]]
     value_coder_impl = self._value_coder.get_impl()
     key_coder_impl = self._key_coder.get_impl()
     for (key, window), values in self._values_by_window.items():
@@ -360,7 +361,7 @@ def encoded_items(self):
       output_stream = create_OutputStream()
       for value in values:
         value_coder_impl.encode_to_stream(value, output_stream, True)
-      yield encoded_key, encoded_window, output_stream.get()
+      yield encoded_key, encoded_window, output_stream.get(), len(values)
 
 
 class GenericNonMergingWindowFn(window.NonMergingWindowFn):
@@ -979,7 +980,7 @@ def commit_side_inputs_to_state(
         elements_by_window.append(element_data)
 
       if func_spec.urn == common_urns.side_inputs.ITERABLE.urn:
-        for _, window, elements_data in elements_by_window.encoded_items():
+        for _, window, elements_data, _ in elements_by_window.encoded_items():
           state_key = beam_fn_api_pb2.StateKey(
               iterable_side_input=beam_fn_api_pb2.StateKey.IterableSideInput(
                   transform_id=consuming_transform_id,
@@ -987,7 +988,11 @@ def commit_side_inputs_to_state(
                   window=window))
           self.state_servicer.append_raw(state_key, elements_data)
       elif func_spec.urn == common_urns.side_inputs.MULTIMAP.urn:
-        for key, window, elements_data in elements_by_window.encoded_items():
+        # TODO(robertwb): Consider computing these lazily on demand rather than
+        # anticipating all potentail state requests which will be more cpu and
+        # memory efficient for large side inputs.
+        for (key, window, elements_data,
+             elements_count) in elements_by_window.encoded_items():
           state_key = beam_fn_api_pb2.StateKey(
               multimap_side_input=beam_fn_api_pb2.StateKey.MultimapSideInput(
                   transform_id=consuming_transform_id,
@@ -995,6 +1000,25 @@ def commit_side_inputs_to_state(
                   window=window,
                   key=key))
           self.state_servicer.append_raw(state_key, elements_data)
+
+          key_iter_state_key = beam_fn_api_pb2.StateKey(
+              multimap_keys_side_input=beam_fn_api_pb2.StateKey.
+              MultimapKeysSideInput(
+                  transform_id=consuming_transform_id,
+                  side_input_id=tag,
+                  window=window))
+          self.state_servicer.append_raw(key_iter_state_key, key)
+
+          kv_iter_state_key = beam_fn_api_pb2.StateKey(
+              multimap_keys_values_side_input=beam_fn_api_pb2.StateKey.
+              MultimapKeysValuesSideInput(
+                  transform_id=consuming_transform_id,
+                  side_input_id=tag,
+                  window=window))
+          self.state_servicer.append_raw(
+              kv_iter_state_key,
+              # KV<K, Iterable<V>> encoding.
+              key + struct.pack('>i', elements_count) + elements_data)
       else:
         raise ValueError("Unknown access pattern: '%s'" % func_spec.urn)
 
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
index 098e16933b73..a736288dc62d 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/fn_runner.py
@@ -224,17 +224,26 @@ def run_via_runner_api(self, pipeline_proto, options):
     return self.run_stages(stage_context, stages)
 
   def embed_default_docker_image(self, pipeline_proto):
+    """Updates the pipeline proto to execute transforms that would normally
+    be executed in the default docker image for this SDK to execute inline
+    via the "embedded" environment.
+    """
     # Context is unused for these types.
     embedded_env = environments.EmbeddedPythonEnvironment.default(
     ).to_runner_api(None)  # type: ignore[arg-type]
     docker_env = environments.DockerEnvironment.from_container_image(
         environments.DockerEnvironment.default_docker_image()).to_runner_api(
             None)  # type: ignore[arg-type]
-    for env_id, env in pipeline_proto.components.environments.items():
-      if env == docker_env:
-        docker_env_id = env_id
-        break
-    else:
+
+    def is_this_python_docker_env(env):
+      return any(
+          e == docker_env for e in environments.expand_anyof_environments(env))
+
+    python_docker_environments = set(
+        env_id
+        for (env_id, env) in pipeline_proto.components.environments.items()
+        if is_this_python_docker_env(env))
+    if not python_docker_environments:
       # No matching docker environments.
       return pipeline_proto
 
@@ -243,13 +252,15 @@ def embed_default_docker_image(self, pipeline_proto):
         embedded_env_id = env_id
         break
     else:
-      # No existing embedded environment.
-      pipeline_proto.components.environments[docker_env_id].CopyFrom(
+      # No existing embedded environment. Create one.
+      embedded_env_id = "python_embedded_env"
+      while embedded_env_id in pipeline_proto.components.environments:
+        embedded_env_id += '_'
+      pipeline_proto.components.environments[embedded_env_id].CopyFrom(
           embedded_env)
-      return pipeline_proto
 
     for transform in pipeline_proto.components.transforms.values():
-      if transform.environment_id == docker_env_id:
+      if transform.environment_id in python_docker_environments:
         transform.environment_id = embedded_env_id
     return pipeline_proto
 
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py
index b11c8349909c..11c8ecd84f21 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner/worker_handlers.py
@@ -87,6 +87,10 @@
 # Time-based flush is enabled in the fn_api_runner by default.
 DATA_BUFFER_TIME_LIMIT_MS = 1000
 
+FNAPI_RUNNER_CAPABILITIES = frozenset([
+    common_urns.runner_protocols.MULTIMAP_KEYS_VALUES_SIDE_INPUT.urn,
+])
+
 _LOGGER = logging.getLogger(__name__)
 
 T = TypeVar('T')
@@ -363,6 +367,7 @@ def __init__(self,
     self.data_conn = self.data_plane_handler
     state_cache = StateCache(STATE_CACHE_SIZE_MB * MB_TO_BYTES)
     self.bundle_processor_cache = sdk_worker.BundleProcessorCache(
+        FNAPI_RUNNER_CAPABILITIES,
         SingletonStateHandlerFactory(
             sdk_worker.GlobalCachingStateHandler(state_cache, state)),
         data_plane.InMemoryDataChannelFactory(
@@ -433,6 +438,7 @@ def GetProvisionInfo(self, request, context=None):
       info.control_endpoint.CopyFrom(worker.control_api_service_descriptor())
     else:
       info = self._base_info
+    info.runner_capabilities[:] = FNAPI_RUNNER_CAPABILITIES
     return beam_provision_api_pb2.GetProvisionInfoResponse(info=info)
 
 
@@ -663,7 +669,8 @@ def start_worker(self):
         self.control_address,
         state_cache_size=self._state_cache_size,
         data_buffer_time_limit_ms=self._data_buffer_time_limit_ms,
-        worker_id=self.worker_id)
+        worker_id=self.worker_id,
+        runner_capabilities=FNAPI_RUNNER_CAPABILITIES)
     self.worker_thread = threading.Thread(
         name='run_worker', target=self.worker.run)
     self.worker_thread.daemon = True
@@ -879,6 +886,18 @@ def get_worker_handlers(
       environment_id = next(iter(self._environments.keys()))
     environment = self._environments[environment_id]
 
+    if environment.urn == common_urns.environments.ANYOF.urn:
+      payload = beam_runner_api_pb2.AnyOfEnvironmentPayload.FromString(
+          environment.payload)
+      env_rankings = {
+          python_urns.EMBEDDED_PYTHON: 10,
+          common_urns.environments.EXTERNAL.urn: 5,
+          common_urns.environments.DOCKER.urn: 1,
+      }
+      environment = sorted(
+          payload.environments,
+          key=lambda env: env_rankings.get(env.urn, -1))[-1]
+
     # assume all environments except EMBEDDED_PYTHON use gRPC.
     if environment.urn == python_urns.EMBEDDED_PYTHON:
       # special case for EmbeddedWorkerHandler: there's no need for a gRPC
@@ -933,6 +952,16 @@ def get_worker(self, worker_id):
 
 class StateServicer(beam_fn_api_pb2_grpc.BeamFnStateServicer,
                     sdk_worker.StateHandler):
+  _SUPPORTED_STATE_TYPES = frozenset([
+      'runner',
+      'multimap_side_input',
+      'multimap_keys_side_input',
+      'multimap_keys_values_side_input',
+      'iterable_side_input',
+      'bag_user_state',
+      'multimap_user_state'
+  ])
+
   class CopyOnWriteState(object):
     def __init__(self, underlying):
       # type: (DefaultDict[bytes, Buffer]) -> None
@@ -1026,6 +1055,11 @@ def get_raw(self,
       continuation_token=None  # type: Optional[bytes]
               ):
     # type: (...) -> Tuple[bytes, Optional[bytes]]
+
+    if state_key.WhichOneof('type') not in self._SUPPORTED_STATE_TYPES:
+      raise NotImplementedError(
+          'Unknown state type: ' + state_key.WhichOneof('type'))
+
     with self._lock:
       full_state = self._state[self._to_key(state_key)]
       if self._use_continuation_tokens:
@@ -1092,24 +1126,27 @@ def State(self,
     # Note that this eagerly mutates state, assuming any failures are fatal.
     # Thus it is safe to ignore instruction_id.
     for request in request_stream:
-      request_type = request.WhichOneof('request')
-      if request_type == 'get':
-        data, continuation_token = self._state.get_raw(
-            request.state_key, request.get.continuation_token)
-        yield beam_fn_api_pb2.StateResponse(
-            id=request.id,
-            get=beam_fn_api_pb2.StateGetResponse(
-                data=data, continuation_token=continuation_token))
-      elif request_type == 'append':
-        self._state.append_raw(request.state_key, request.append.data)
-        yield beam_fn_api_pb2.StateResponse(
-            id=request.id, append=beam_fn_api_pb2.StateAppendResponse())
-      elif request_type == 'clear':
-        self._state.clear(request.state_key)
-        yield beam_fn_api_pb2.StateResponse(
-            id=request.id, clear=beam_fn_api_pb2.StateClearResponse())
-      else:
-        raise NotImplementedError('Unknown state request: %s' % request_type)
+      try:
+        request_type = request.WhichOneof('request')
+        if request_type == 'get':
+          data, continuation_token = self._state.get_raw(
+              request.state_key, request.get.continuation_token)
+          yield beam_fn_api_pb2.StateResponse(
+              id=request.id,
+              get=beam_fn_api_pb2.StateGetResponse(
+                  data=data, continuation_token=continuation_token))
+        elif request_type == 'append':
+          self._state.append_raw(request.state_key, request.append.data)
+          yield beam_fn_api_pb2.StateResponse(
+              id=request.id, append=beam_fn_api_pb2.StateAppendResponse())
+        elif request_type == 'clear':
+          self._state.clear(request.state_key)
+          yield beam_fn_api_pb2.StateResponse(
+              id=request.id, clear=beam_fn_api_pb2.StateClearResponse())
+        else:
+          raise NotImplementedError('Unknown state request: %s' % request_type)
+      except Exception as exn:
+        yield beam_fn_api_pb2.StateResponse(id=request.id, error=str(exn))
 
 
 class SingletonStateHandlerFactory(sdk_worker.StateHandlerFactory):
diff --git a/sdks/python/apache_beam/runners/portability/local_job_service.py b/sdks/python/apache_beam/runners/portability/local_job_service.py
index 91ddb3fced15..6966e66d2c64 100644
--- a/sdks/python/apache_beam/runners/portability/local_job_service.py
+++ b/sdks/python/apache_beam/runners/portability/local_job_service.py
@@ -28,7 +28,9 @@
 import time
 import traceback
 from typing import TYPE_CHECKING
+from typing import Any
 from typing import List
+from typing import Mapping
 from typing import Optional
 
 import grpc
@@ -43,6 +45,7 @@
 from apache_beam.portability.api import beam_job_api_pb2
 from apache_beam.portability.api import beam_job_api_pb2_grpc
 from apache_beam.portability.api import beam_provision_api_pb2
+from apache_beam.portability.api import beam_runner_api_pb2
 from apache_beam.portability.api import endpoints_pb2
 from apache_beam.runners.job import utils as job_utils
 from apache_beam.runners.portability import abstract_job_service
@@ -51,11 +54,11 @@
 from apache_beam.runners.portability.fn_api_runner import fn_runner
 from apache_beam.runners.portability.fn_api_runner import worker_handlers
 from apache_beam.runners.worker.log_handler import LOGENTRY_TO_LOG_LEVEL_MAP
+from apache_beam.transforms import environments
 from apache_beam.utils import thread_pool_executor
 
 if TYPE_CHECKING:
   from google.protobuf import struct_pb2  # pylint: disable=ungrouped-imports
-  from apache_beam.portability.api import beam_runner_api_pb2
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -96,10 +99,8 @@ def create_beam_job(self,
     # type: (...) -> BeamJob
     self._artifact_service.register_job(
         staging_token=preparation_id,
-        dependency_sets={
-            id: env.dependencies
-            for (id, env) in pipeline.components.environments.items()
-        })
+        dependency_sets=_extract_dependency_sets(
+            pipeline.components.environments))
     provision_info = fn_runner.ExtendedProvisionInfo(
         beam_provision_api_pb2.ProvisionInfo(pipeline_options=options),
         self._staging_dir,
@@ -321,12 +322,9 @@ def _invoke_runner(self):
 
   def _update_dependencies(self):
     try:
-      for env_id, deps in self._artifact_service.resolved_deps(
-          self._job_id, timeout=0).items():
-        # Slice assignment not supported for repeated fields.
-        env = self._pipeline_proto.components.environments[env_id]
-        del env.dependencies[:]
-        env.dependencies.extend(deps)
+      _update_dependency_sets(
+          self._pipeline_proto.components.environments,
+          self._artifact_service.resolved_deps(self._job_id, timeout=0))
       self._provision_info.provision_info.ClearField('retrieval_token')
     except concurrent.futures.TimeoutError:
       # TODO(https://github.com/apache/beam/issues/20267): Require this once
@@ -457,3 +455,42 @@ def emit(self, record):
 
       # Inform all message consumers.
       self._log_queues.put(msg)
+
+
+def _extract_dependency_sets(
+    envs: Mapping[str, beam_runner_api_pb2.Environment]
+) -> Mapping[Any, List[beam_runner_api_pb2.ArtifactInformation]]:
+  """Expands the set of environments into a mapping of (opaque) keys to
+  dependency sets.  This is not 1:1 in the case of AnyOf environments.
+
+  The values can then be resolved and the mapping passed back to
+  _update_dependency_sets to update the dependencies in the original protos.
+  """
+  def dependencies_iter():
+    for env_id, env in envs.items():
+      for ix, sub_env in enumerate(environments.expand_anyof_environments(env)):
+        yield (env_id, ix), sub_env.dependencies
+
+  return dict(dependencies_iter())
+
+
+def _update_dependency_sets(
+    envs: Mapping[str, beam_runner_api_pb2.Environment],
+    resolved_deps: Mapping[Any, List[beam_runner_api_pb2.ArtifactInformation]]):
+  """Takes the mapping of beam Environments (originally passed to
+  `_extract_dependency_sets`) and a set of (key-wise) updated dependencies,
+  and updates the original environment protos to contain the updated
+  dependencies.
+  """
+  for env_id, env in envs.items():
+    new_envs = []
+    for ix, sub_env in enumerate(environments.expand_anyof_environments(env)):
+      # Slice assignment not supported for repeated fields.
+      del sub_env.dependencies[:]
+      sub_env.dependencies.extend(resolved_deps[env_id, ix])
+      new_envs.append(sub_env)
+    if len(new_envs) == 1:
+      envs[env_id].CopyFrom(new_envs[0])
+    else:
+      envs[env_id].CopyFrom(
+          environments.AnyOfEnvironment.create_proto(new_envs))
diff --git a/sdks/python/apache_beam/runners/portability/sdk_container_builder.py b/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
index ea86eafcc34a..d5d1bca981d4 100644
--- a/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
+++ b/sdks/python/apache_beam/runners/portability/sdk_container_builder.py
@@ -213,7 +213,11 @@ def __init__(self, options):
     if credentials:
       self._storage_client = storage.Client(
           credentials=credentials.get_google_auth_credentials(),
-          project=self._google_cloud_options.project)
+          project=self._google_cloud_options.project,
+          extra_headers={
+              "User-Agent": "apache-beam/%s (GPN:Beam)" %
+              beam_version.__version__
+          })
     else:
       self._storage_client = storage.Client.create_anonymous_client()
     self._cloudbuild_client = cloudbuild.CloudbuildV1(
diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor.py b/sdks/python/apache_beam/runners/worker/bundle_processor.py
index 7ff0ad258bc2..cf2b61d48b50 100644
--- a/sdks/python/apache_beam/runners/worker/bundle_processor.py
+++ b/sdks/python/apache_beam/runners/worker/bundle_processor.py
@@ -378,12 +378,18 @@ def __reduce__(self):
 
 
 class StateBackedSideInputMap(object):
+
+  _BULK_READ_LIMIT = 100
+  _BULK_READ_FULLY = "fully"
+  _BULK_READ_PARTIALLY = "partially"
+
   def __init__(self,
                state_handler,  # type: sdk_worker.CachingStateHandler
                transform_id,  # type: str
                tag,  # type: Optional[str]
                side_input_data,  # type: pvalue.SideInputData
-               coder  # type: WindowedValueCoder
+               coder,  # type: WindowedValueCoder
+               use_bulk_read = False,  # type: bool
               ):
     # type: (...) -> None
     self._state_handler = state_handler
@@ -394,6 +400,7 @@ def __init__(self,
     self._target_window_coder = coder.window_coder
     # TODO(robertwb): Limit the cache size.
     self._cache = {}  # type: Dict[BoundedWindow, Any]
+    self._use_bulk_read = use_bulk_read
 
   def __getitem__(self, window):
     target_window = self._side_input_data.window_mapping_fn(window)
@@ -417,12 +424,55 @@ def __getitem__(self, window):
                 side_input_id=self._tag,
                 window=self._target_window_coder.encode(target_window),
                 key=b''))
+        kv_iter_state_key = beam_fn_api_pb2.StateKey(
+            multimap_keys_values_side_input=beam_fn_api_pb2.StateKey.
+            MultimapKeysValuesSideInput(
+                transform_id=self._transform_id,
+                side_input_id=self._tag,
+                window=self._target_window_coder.encode(target_window)))
         cache = {}
-        key_coder_impl = self._element_coder.key_coder().get_impl()
+        key_coder = self._element_coder.key_coder()
+        key_coder_impl = key_coder.get_impl()
         value_coder = self._element_coder.value_coder()
+        use_bulk_read = self._use_bulk_read
 
         class MultiMap(object):
+          _bulk_read = None
+          _lock = threading.Lock()
+
           def __getitem__(self, key):
+            if use_bulk_read:
+              if self._bulk_read is None:
+                with self._lock:
+                  if self._bulk_read is None:
+                    try:
+                      # Attempt to bulk read the key-values over the iterable
+                      # protocol which, if supported, can be much more efficient
+                      # than point lookups if it fits into memory.
+                      for ix, (k, vs) in enumerate(_StateBackedIterable(
+                          state_handler,
+                          kv_iter_state_key,
+                          coders.TupleCoder(
+                              (key_coder, coders.IterableCoder(value_coder))))):
+                        cache[k] = vs
+                        if ix > StateBackedSideInputMap._BULK_READ_LIMIT:
+                          self._bulk_read = (
+                              StateBackedSideInputMap._BULK_READ_PARTIALLY)
+                          break
+                      else:
+                        # We reached the end of the iteration without breaking.
+                        self._bulk_read = (
+                            StateBackedSideInputMap._BULK_READ_FULLY)
+                    except Exception:
+                      _LOGGER.error(
+                          "Iterable access of map side inputs unsupported.",
+                          exc_info=True)
+                      self._bulk_read = (
+                          StateBackedSideInputMap._BULK_READ_PARTIALLY)
+
+              if (self._bulk_read == StateBackedSideInputMap._BULK_READ_FULLY):
+                return cache.get(key, [])
+
             if key not in cache:
               keyed_state_key = beam_fn_api_pb2.StateKey()
               keyed_state_key.CopyFrom(state_key)
@@ -430,6 +480,7 @@ def __getitem__(self, key):
                   key_coder_impl.encode_nested(key))
               cache[key] = _StateBackedIterable(
                   state_handler, keyed_state_key, value_coder)
+
             return cache[key]
 
           def __reduce__(self):
@@ -869,6 +920,7 @@ class BundleProcessor(object):
   """ A class for processing bundles of elements. """
 
   def __init__(self,
+               runner_capabilities,  # type: FrozenSet[str]
                process_bundle_descriptor,  # type: beam_fn_api_pb2.ProcessBundleDescriptor
                state_handler,  # type: sdk_worker.CachingStateHandler
                data_channel_factory,  # type: data_plane.DataChannelFactory
@@ -879,11 +931,14 @@ def __init__(self,
     """Initialize a bundle processor.
 
     Args:
+      runner_capabilities (``FrozenSet[str]``): The set of capabilities of the
+        runner with which we will be interacting
       process_bundle_descriptor (``beam_fn_api_pb2.ProcessBundleDescriptor``):
         a description of the stage that this ``BundleProcessor``is to execute.
       state_handler (CachingStateHandler).
       data_channel_factory (``data_plane.DataChannelFactory``).
     """
+    self.runner_capabilities = runner_capabilities
     self.process_bundle_descriptor = process_bundle_descriptor
     self.state_handler = state_handler
     self.data_channel_factory = data_channel_factory
@@ -925,12 +980,14 @@ def create_execution_tree(
   ):
     # type: (...) -> collections.OrderedDict[str, operations.DoOperation]
     transform_factory = BeamTransformFactory(
+        self.runner_capabilities,
         descriptor,
         self.data_channel_factory,
         self.counter_factory,
         self.state_sampler,
         self.state_handler,
-        self.data_sampler)
+        self.data_sampler,
+    )
 
     self.timers_info = transform_factory.extract_timers_info()
 
@@ -1216,6 +1273,7 @@ class ExecutionContext:
 class BeamTransformFactory(object):
   """Factory for turning transform_protos into executable operations."""
   def __init__(self,
+               runner_capabilities,  # type: FrozenSet[str]
                descriptor,  # type: beam_fn_api_pb2.ProcessBundleDescriptor
                data_channel_factory,  # type: data_plane.DataChannelFactory
                counter_factory,  # type: counters.CounterFactory
@@ -1223,6 +1281,7 @@ def __init__(self,
                state_handler,  # type: sdk_worker.CachingStateHandler
                data_sampler,  # type: Optional[data_sampler.DataSampler]
               ):
+    self.runner_capabilities = runner_capabilities
     self.descriptor = descriptor
     self.data_channel_factory = data_channel_factory
     self.counter_factory = counter_factory
@@ -1648,8 +1707,11 @@ def _create_pardo_operation(
             transform_id,
             tag,
             si,
-            input_tags_to_coders[tag]) for tag,
-        si in tagged_side_inputs
+            input_tags_to_coders[tag],
+            use_bulk_read=(
+                common_urns.runner_protocols.MULTIMAP_KEYS_VALUES_SIDE_INPUT.urn
+                in factory.runner_capabilities))
+        for (tag, si) in tagged_side_inputs
     ]
   else:
     side_input_maps = []
diff --git a/sdks/python/apache_beam/runners/worker/bundle_processor_test.py b/sdks/python/apache_beam/runners/worker/bundle_processor_test.py
index 292b8431063c..dafb4dbd4bf0 100644
--- a/sdks/python/apache_beam/runners/worker/bundle_processor_test.py
+++ b/sdks/python/apache_beam/runners/worker/bundle_processor_test.py
@@ -267,7 +267,7 @@ def test_disabled_by_default(self):
     """
     descriptor = beam_fn_api_pb2.ProcessBundleDescriptor()
     descriptor.pcollections['a'].unique_name = 'a'
-    _ = BundleProcessor(descriptor, None, None)
+    _ = BundleProcessor(set(), descriptor, None, None)
     self.assertEqual(len(descriptor.transforms), 0)
 
   def test_can_sample(self):
@@ -301,7 +301,7 @@ def test_can_sample(self):
       # Create and process a fake bundle. The instruction id doesn't matter
       # here.
       processor = BundleProcessor(
-          descriptor, None, None, data_sampler=data_sampler)
+          set(), descriptor, None, None, data_sampler=data_sampler)
       processor.process_bundle('instruction_id')
 
       samples = data_sampler.wait_for_samples([PCOLLECTION_ID])
@@ -377,7 +377,7 @@ def test_can_sample_exceptions(self):
       # Create and process a fake bundle. The instruction id doesn't matter
       # here.
       processor = BundleProcessor(
-          descriptor, None, None, data_sampler=data_sampler)
+          set(), descriptor, None, None, data_sampler=data_sampler)
 
       with self.assertRaisesRegex(RuntimeError, 'expected exception'):
         processor.process_bundle('instruction_id')
diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py
index 9eb9299cac39..2cf7dff9d57f 100644
--- a/sdks/python/apache_beam/runners/worker/log_handler_test.py
+++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py
@@ -286,7 +286,7 @@ def test_extracts_transform_id_during_exceptions(self):
 
     # Create and process a fake bundle. The instruction id doesn't matter
     # here.
-    processor = BundleProcessor(descriptor, None, None)
+    processor = BundleProcessor(set(), descriptor, None, None)
 
     with self.assertRaisesRegex(RuntimeError, 'expected exception'):
       processor.process_bundle('instruction_id')
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py
index bfd6544d802b..a64fb4c967a6 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py
@@ -174,6 +174,7 @@ def __init__(
       # Unrecoverable SDK harness initialization error (if any)
       # that should be reported to the runner when proocessing the first bundle.
       deferred_exception=None, # type: Optional[Exception]
+      runner_capabilities=frozenset(), # type: FrozenSet[str]
   ):
     # type: (...) -> None
     self._alive = True
@@ -202,6 +203,7 @@ def __init__(
         self._state_cache, credentials)
     self._profiler_factory = profiler_factory
     self.data_sampler = data_sampler
+    self.runner_capabilities = runner_capabilities
 
     def default_factory(id):
       # type: (str) -> beam_fn_api_pb2.ProcessBundleDescriptor
@@ -212,6 +214,7 @@ def default_factory(id):
     self._fns = KeyedDefaultDict(default_factory)
     # BundleProcessor cache across all workers.
     self._bundle_processor_cache = BundleProcessorCache(
+        self.runner_capabilities,
         state_handler_factory=self._state_handler_factory,
         data_channel_factory=self._data_channel_factory,
         fns=self._fns,
@@ -419,12 +422,14 @@ class BundleProcessorCache(object):
 
   def __init__(
       self,
+      runner_capabilities,  # type: FrozenSet[str]
       state_handler_factory,  # type: StateHandlerFactory
       data_channel_factory,  # type: data_plane.DataChannelFactory
       fns,  # type: MutableMapping[str, beam_fn_api_pb2.ProcessBundleDescriptor]
       data_sampler=None,  # type: Optional[data_sampler.DataSampler]
   ):
     # type: (...) -> None
+    self.runner_capabilities = runner_capabilities
     self.fns = fns
     self.state_handler_factory = state_handler_factory
     self.data_channel_factory = data_channel_factory
@@ -484,10 +489,18 @@ def get(self, instruction_id, bundle_descriptor_id):
         pass
 
     # Make sure we instantiate the processor while not holding the lock.
+
+    # Reduce risks of concurrent modifications of the same protos
+    # captured in bundle descriptor when the same bundle descriptor is used
+    # in different instructions.
+    pbd = beam_fn_api_pb2.ProcessBundleDescriptor()
+    pbd.MergeFrom(self.fns[bundle_descriptor_id])
+
     processor = bundle_processor.BundleProcessor(
-        self.fns[bundle_descriptor_id],
+        self.runner_capabilities,
+        pbd,
         self.state_handler_factory.create_state_handler(
-            self.fns[bundle_descriptor_id].state_api_service_descriptor),
+            pbd.state_api_service_descriptor),
         self.data_channel_factory,
         self.data_sampler)
     with self._lock:
@@ -583,11 +596,12 @@ def _schedule_periodic_shutdown(self):
     # type: () -> None
     def shutdown_inactive_bundle_processors():
       # type: () -> None
-      for descriptor_id, last_access_time in self.last_access_times.items():
-        if (time.time() - last_access_time >
-            DEFAULT_BUNDLE_PROCESSOR_CACHE_SHUTDOWN_THRESHOLD_S):
-          BundleProcessorCache._shutdown_cached_bundle_processors(
-              self.cached_bundle_processors[descriptor_id])
+      with self._lock:
+        for descriptor_id, last_access_time in self.last_access_times.items():
+          if (time.time() - last_access_time >
+              DEFAULT_BUNDLE_PROCESSOR_CACHE_SHUTDOWN_THRESHOLD_S):
+            BundleProcessorCache._shutdown_cached_bundle_processors(
+                self.cached_bundle_processors[descriptor_id])
 
     self.periodic_shutdown = PeriodicThread(
         DEFAULT_BUNDLE_PROCESSOR_CACHE_SHUTDOWN_THRESHOLD_S,
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
index 1af0071edc14..cd49c69a80aa 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py
@@ -103,10 +103,9 @@ def create_harness(environment, dry_run=False):
   pickle_library = sdk_pipeline_options.view_as(SetupOptions).pickle_library
   pickler.set_library(pickle_library)
 
-  if 'SEMI_PERSISTENT_DIRECTORY' in environment:
-    semi_persistent_directory = environment['SEMI_PERSISTENT_DIRECTORY']
-  else:
-    semi_persistent_directory = None
+  semi_persistent_directory = environment.get('SEMI_PERSISTENT_DIRECTORY', None)
+  runner_capabilities = frozenset(
+      environment.get('RUNNER_CAPABILITIES', '').split())
 
   _LOGGER.info('semi_persistent_directory: %s', semi_persistent_directory)
   _worker_id = environment.get('WORKER_ID', None)
@@ -167,7 +166,8 @@ def create_harness(environment, dry_run=False):
           sdk_pipeline_options.view_as(ProfilingOptions)),
       enable_heap_dump=enable_heap_dump,
       data_sampler=data_sampler,
-      deferred_exception=deferred_exception)
+      deferred_exception=deferred_exception,
+      runner_capabilities=runner_capabilities)
   return fn_log_handler, sdk_harness, sdk_pipeline_options
 
 
diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
index 8570c5a7722c..4e202910345c 100644
--- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
+++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py
@@ -126,7 +126,7 @@ def test_fn_registration(self):
 
   def test_inactive_bundle_processor_returns_empty_progress_response(self):
     bundle_processor = mock.MagicMock()
-    bundle_processor_cache = BundleProcessorCache(None, None, {})
+    bundle_processor_cache = BundleProcessorCache(None, None, None, {})
     bundle_processor_cache.activate('instruction_id')
     worker = SdkWorker(bundle_processor_cache)
     split_request = beam_fn_api_pb2.InstructionRequest(
@@ -153,7 +153,7 @@ def test_inactive_bundle_processor_returns_empty_progress_response(self):
 
   def test_failed_bundle_processor_returns_failed_progress_response(self):
     bundle_processor = mock.MagicMock()
-    bundle_processor_cache = BundleProcessorCache(None, None, {})
+    bundle_processor_cache = BundleProcessorCache(None, None, None, {})
     bundle_processor_cache.activate('instruction_id')
     worker = SdkWorker(bundle_processor_cache)
 
@@ -172,7 +172,7 @@ def test_failed_bundle_processor_returns_failed_progress_response(self):
 
   def test_inactive_bundle_processor_returns_empty_split_response(self):
     bundle_processor = mock.MagicMock()
-    bundle_processor_cache = BundleProcessorCache(None, None, {})
+    bundle_processor_cache = BundleProcessorCache(None, None, None, {})
     bundle_processor_cache.activate('instruction_id')
     worker = SdkWorker(bundle_processor_cache)
     split_request = beam_fn_api_pb2.InstructionRequest(
@@ -258,7 +258,7 @@ def test_harness_monitoring_infos_and_metadata(self):
 
   def test_failed_bundle_processor_returns_failed_split_response(self):
     bundle_processor = mock.MagicMock()
-    bundle_processor_cache = BundleProcessorCache(None, None, {})
+    bundle_processor_cache = BundleProcessorCache(None, None, None, {})
     bundle_processor_cache.activate('instruction_id')
     worker = SdkWorker(bundle_processor_cache)
 
diff --git a/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py b/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py
index 1ff46a3f7d19..d1da4667dcb8 100644
--- a/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py
+++ b/sdks/python/apache_beam/testing/load_tests/load_test_metrics_utils.py
@@ -33,6 +33,8 @@
 import logging
 import time
 import uuid
+from typing import Any
+from typing import Dict
 from typing import List
 from typing import Mapping
 from typing import Optional
@@ -45,6 +47,7 @@
 from apache_beam.metrics import Metrics
 from apache_beam.metrics.metric import MetricResults
 from apache_beam.metrics.metric import MetricsFilter
+from apache_beam.runners.dataflow.dataflow_runner import DataflowPipelineResult
 from apache_beam.runners.runner import PipelineResult
 from apache_beam.transforms.window import TimestampedValue
 from apache_beam.utils.timestamp import Timestamp
@@ -65,6 +68,7 @@
 SUBMIT_TIMESTAMP_LABEL = 'timestamp'
 METRICS_TYPE_LABEL = 'metric'
 VALUE_LABEL = 'value'
+JOB_ID_LABEL = 'job_id'
 
 SCHEMA = [{
     'name': ID_LABEL, 'field_type': 'STRING', 'mode': 'REQUIRED'
@@ -80,6 +84,8 @@
               'mode': 'REQUIRED'
           }, {
               'name': VALUE_LABEL, 'field_type': 'FLOAT', 'mode': 'REQUIRED'
+          }, {
+              'name': JOB_ID_LABEL, 'field_type': 'STRING', 'mode': 'NULLABLE'
           }]
 
 _LOGGER = logging.getLogger(__name__)
@@ -254,13 +260,27 @@ def publish_metrics(
     # Under each key there is list of objects of each metric type. It is
     # required to prepare metrics for publishing purposes. Expected is to have
     # a list of dictionaries matching the schema.
+
     insert_dicts = self._prepare_all_metrics(metrics, metric_id)
 
     insert_dicts += self._prepare_extra_metrics(metric_id, extra_metrics)
+
+    # Add job id for dataflow jobs for easier debugging.
+    job_id = None
+    if isinstance(result, DataflowPipelineResult):
+      job_id = result.job_id()
+      self._add_job_id_to_metrics(insert_dicts, job_id)
+
     if len(insert_dicts) > 0:
       for publisher in self.publishers:
         publisher.publish(insert_dicts)
 
+  def _add_job_id_to_metrics(self, metrics: List[Dict[str, Any]],
+                             job_id) -> List[Dict[str, Any]]:
+    for metric in metrics:
+      metric[JOB_ID_LABEL] = job_id
+    return metrics
+
   def _prepare_extra_metrics(
       self, metric_id: str, extra_metrics: Optional[dict] = None):
     ts = time.time()
@@ -479,6 +499,12 @@ def _get_or_create_table(self, bq_schemas, dataset):
       table = bigquery.Table(table_ref, schema=bq_schemas)
       self._bq_table = self._client.create_table(table)
 
+  def _update_schema(self):
+    table_schema = self._bq_table.schema
+    if self.schema and len(table_schema) != self.schema:
+      self._bq_table.schema = self._prepare_schema()
+      self._bq_table = self._client.update_table(self._bq_table, ["schema"])
+
   def _get_dataset(self, dataset_name):
     bq_dataset_ref = self._client.dataset(dataset_name)
     try:
@@ -490,6 +516,8 @@ def _get_dataset(self, dataset_name):
     return bq_dataset
 
   def save(self, results):
+    # update schema if needed
+    self._update_schema()
     return self._client.insert_rows(self._bq_table, results)
 
 
diff --git a/sdks/python/apache_beam/transforms/environments.py b/sdks/python/apache_beam/transforms/environments.py
index 109fcb825347..b2fbe87a729a 100644
--- a/sdks/python/apache_beam/transforms/environments.py
+++ b/sdks/python/apache_beam/transforms/environments.py
@@ -61,6 +61,7 @@
 
 __all__ = [
     'Environment',
+    'AnyOfEnvironment',
     'DefaultEnvironment',
     'DockerEnvironment',
     'ProcessEnvironment',
@@ -584,6 +585,24 @@ def from_options(cls, options):
         resource_hints=resource_hints_from_options(options))
 
 
+def expand_anyof_environments(env_proto):
+  if env_proto.urn == common_urns.environments.ANYOF.urn:
+    for alt in beam_runner_api_pb2.AnyOfEnvironmentPayload.FromString(
+        env_proto.payload).environments:
+      yield from expand_anyof_environments(alt)
+  else:
+    yield env_proto
+
+
+def resolve_anyof_environment(env_proto, *preferred_types):
+  all_environments = list(expand_anyof_environments(env_proto))
+  for preferred_type in preferred_types:
+    for env in all_environments:
+      if env.urn == preferred_type:
+        return env
+  return all_environments[0]
+
+
 @Environment.register_urn(python_urns.EMBEDDED_PYTHON, None)
 class EmbeddedPythonEnvironment(Environment):
   def to_runner_api_parameter(self, context):
@@ -796,6 +815,45 @@ def from_command_string(cls, command_string):
         command_string, capabilities=python_sdk_capabilities(), artifacts=())
 
 
+@Environment.register_urn(
+    common_urns.environments.ANYOF.urn,
+    beam_runner_api_pb2.AnyOfEnvironmentPayload)
+class AnyOfEnvironment(Environment):
+  def __init__(self, environments):
+    self._environments = environments
+
+  def to_runner_api_parameter(self, context):
+    # type: (PipelineContext) -> Tuple[str, beam_runner_api_pb2.AnyOfEnvironmentPayload]
+    return (
+        common_urns.environments.ANYOF.urn,
+        beam_runner_api_pb2.AnyOfEnvironmentPayload(
+            environments=[
+                env.to_runner_api(context) for env in self._environments
+            ]))
+
+  @staticmethod
+  def from_runner_api_parameter(payload,  # type: beam_runner_api_pb2.AnyOfEnvironmentPayload
+      capabilities,  # type: Iterable[str]
+      artifacts,  # type: Iterable[beam_runner_api_pb2.ArtifactInformation]
+      resource_hints,  # type: Mapping[str, bytes]
+      context  # type: PipelineContext
+                                ):
+    # type: (...) -> AnyOfEnvironment
+    return AnyOfEnvironment([
+        Environment.from_runner_api(env, context)
+        for env in payload.environments
+    ])
+
+  @staticmethod
+  def create_proto(
+      environments: Iterable[beam_runner_api_pb2.Environment]
+  ) -> beam_runner_api_pb2.Environment:
+    return beam_runner_api_pb2.Environment(
+        urn=common_urns.environments.ANYOF.urn,
+        payload=beam_runner_api_pb2.AnyOfEnvironmentPayload(
+            environments=environments).SerializeToString())
+
+
 class PyPIArtifactRegistry(object):
   _registered_artifacts = set()  # type: Set[Tuple[str, str]]
 
diff --git a/sdks/python/apache_beam/transforms/external.py b/sdks/python/apache_beam/transforms/external.py
index 0d0b6f1e7be2..71dfc545204a 100644
--- a/sdks/python/apache_beam/transforms/external.py
+++ b/sdks/python/apache_beam/transforms/external.py
@@ -41,8 +41,10 @@
 from apache_beam.portability.api import beam_expansion_api_pb2_grpc
 from apache_beam.portability.api import beam_runner_api_pb2
 from apache_beam.portability.api import external_transforms_pb2
+from apache_beam.portability.api import schema_pb2
 from apache_beam.runners import pipeline_context
 from apache_beam.runners.portability import artifact_service
+from apache_beam.transforms import environments
 from apache_beam.transforms import ptransform
 from apache_beam.typehints import WithTypeHints
 from apache_beam.typehints import native_type_compatibility
@@ -50,6 +52,7 @@
 from apache_beam.typehints.schemas import named_fields_to_schema
 from apache_beam.typehints.schemas import named_tuple_from_schema
 from apache_beam.typehints.schemas import named_tuple_to_schema
+from apache_beam.typehints.schemas import typing_from_runner_api
 from apache_beam.typehints.trivial_inference import instance_to_type
 from apache_beam.typehints.typehints import Union
 from apache_beam.typehints.typehints import UnionConstraint
@@ -372,7 +375,7 @@ def _has_constructor(self):
 # Information regarding a SchemaTransform available in an external SDK.
 SchemaTransformsConfig = namedtuple(
     'SchemaTransformsConfig',
-    ['identifier', 'configuration_schema', 'inputs', 'outputs'])
+    ['identifier', 'configuration_schema', 'inputs', 'outputs', 'description'])
 
 
 class SchemaAwareExternalTransform(ptransform.PTransform):
@@ -443,22 +446,39 @@ def discover_iter(expansion_service, ignore_errors=True):
       discover_response = service.DiscoverSchemaTransform(
           beam_expansion_api_pb2.DiscoverSchemaTransformRequest())
 
-      for identifier in discover_response.schema_transform_configs:
-        proto_config = discover_response.schema_transform_configs[identifier]
-        try:
-          schema = named_tuple_from_schema(proto_config.config_schema)
-        except Exception as exn:
-          if ignore_errors:
+    for identifier in discover_response.schema_transform_configs:
+      proto_config = discover_response.schema_transform_configs[identifier]
+      try:
+        schema = named_tuple_from_schema(proto_config.config_schema)
+      except Exception as exn:
+        if ignore_errors:
+          truncated_schema = schema_pb2.Schema()
+          truncated_schema.CopyFrom(proto_config.config_schema)
+          for field in truncated_schema.fields:
+            try:
+              typing_from_runner_api(field.type)
+            except Exception:
+              if field.type.nullable:
+                # Set it to an empty placeholder type.
+                field.type.CopyFrom(
+                    schema_pb2.FieldType(
+                        nullable=True,
+                        row_type=schema_pb2.RowType(
+                            schema=schema_pb2.Schema())))
+          try:
+            schema = named_tuple_from_schema(truncated_schema)
+          except Exception as exn:
             logging.info("Bad schema for %s: %s", identifier, str(exn)[:250])
             continue
-          else:
-            raise
+        else:
+          raise
 
-        yield SchemaTransformsConfig(
-            identifier=identifier,
-            configuration_schema=schema,
-            inputs=proto_config.input_pcollection_names,
-            outputs=proto_config.output_pcollection_names)
+      yield SchemaTransformsConfig(
+          identifier=identifier,
+          configuration_schema=schema,
+          inputs=proto_config.input_pcollection_names,
+          outputs=proto_config.output_pcollection_names,
+          description=proto_config.description)
 
   @staticmethod
   def discover_config(expansion_service, name):
@@ -721,7 +741,8 @@ def expand(self, pvalueish):
         components=components,
         namespace=self._external_namespace,
         transform=transform_proto,
-        output_coder_requests=output_coders)
+        output_coder_requests=output_coders,
+        pipeline_options=pipeline._options.to_runner_api())
 
     expansion_service = _maybe_use_transform_service(
         self._expansion_service, pipeline.options)
@@ -731,8 +752,9 @@ def expand(self, pvalueish):
       if response.error:
         raise RuntimeError(response.error)
       self._expanded_components = response.components
-      if any(env.dependencies
-             for env in self._expanded_components.environments.values()):
+      if any(e.dependencies
+             for env in self._expanded_components.environments.values()
+             for e in environments.expand_anyof_environments(env)):
         self._expanded_components = self._resolve_artifacts(
             self._expanded_components,
             service.artifact_service(),
@@ -785,12 +807,22 @@ def service(expansion_service):
         yield stub
 
   def _resolve_artifacts(self, components, service, dest):
-    for env in components.environments.values():
-      if env.dependencies:
+    def _resolve_artifacts_for(env):
+      if env.urn == common_urns.environments.ANYOF.urn:
+        env.CopyFrom(
+            environments.AnyOfEnvironment.create_proto([
+                _resolve_artifacts_for(e)
+                for e in environments.expand_anyof_environments(env)
+            ]))
+      elif env.dependencies:
         resolved = list(
             artifact_service.resolve_artifacts(env.dependencies, service, dest))
         del env.dependencies[:]
         env.dependencies.extend(resolved)
+      return env
+
+    for env in components.environments.values():
+      _resolve_artifacts_for(env)
     return components
 
   def _output_to_pvalueish(self, output_dict):
@@ -968,7 +1000,12 @@ def _default_args(self):
     to_stage = ','.join([self._path_to_jar] + sum((
         JavaJarExpansionService._expand_jars(jar)
         for jar in self._classpath or []), []))
-    return ['{{PORT}}', f'--filesToStage={to_stage}']
+    args = ['{{PORT}}', f'--filesToStage={to_stage}']
+    # TODO(robertwb): See if it's possible to scope this per pipeline.
+    # Checks to see if the cache is being used for this server.
+    if subprocess_server.SubprocessServer._cache._live_owners:
+      args.append('--alsoStartLoopbackWorker')
+    return args
 
   def __enter__(self):
     if self._service_count == 0:
@@ -1030,6 +1067,7 @@ def __init__(
       append_args=None):
     path_to_jar = subprocess_server.JavaJarServer.path_to_beam_jar(
         gradle_target, gradle_appendix)
+    self.gradle_target = gradle_target
     super().__init__(
         path_to_jar, extra_args, classpath=classpath, append_args=append_args)
 
diff --git a/sdks/python/apache_beam/transforms/external_schematransform_provider.py b/sdks/python/apache_beam/transforms/external_schematransform_provider.py
new file mode 100644
index 000000000000..fd650087893a
--- /dev/null
+++ b/sdks/python/apache_beam/transforms/external_schematransform_provider.py
@@ -0,0 +1,277 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import re
+from collections import namedtuple
+from typing import Dict
+from typing import List
+from typing import Tuple
+
+from apache_beam.transforms import PTransform
+from apache_beam.transforms.external import BeamJarExpansionService
+from apache_beam.transforms.external import SchemaAwareExternalTransform
+from apache_beam.transforms.external import SchemaTransformsConfig
+from apache_beam.typehints.schemas import named_tuple_to_schema
+from apache_beam.typehints.schemas import typing_from_runner_api
+
+__all__ = ['ExternalSchemaTransform', 'ExternalSchemaTransformProvider']
+
+
+def snake_case_to_upper_camel_case(string):
+  """Convert snake_case to UpperCamelCase"""
+  components = string.split('_')
+  output = ''.join(n.capitalize() for n in components)
+  return output
+
+
+def snake_case_to_lower_camel_case(string):
+  """Convert snake_case to lowerCamelCase"""
+  if len(string) <= 1:
+    return string.lower()
+  upper = snake_case_to_upper_camel_case(string)
+  return upper[0].lower() + upper[1:]
+
+
+def camel_case_to_snake_case(string):
+  """Convert camelCase to snake_case"""
+  arr = []
+  word = []
+  for i, n in enumerate(string):
+    # If seeing an upper letter after a lower letter, we just witnessed a word
+    # If seeing an upper letter and the next letter is lower, we may have just
+    # witnessed an all caps word
+    if n.isupper() and ((i > 0 and string[i - 1].islower()) or
+                        (i + 1 < len(string) and string[i + 1].islower())):
+      arr.append(''.join(word))
+      word = [n.lower()]
+    else:
+      word.append(n.lower())
+  arr.append(''.join(word))
+  return '_'.join(arr).strip('_')
+
+
+# Information regarding a Wrapper parameter.
+ParamInfo = namedtuple('ParamInfo', ['type', 'description', 'original_name'])
+
+
+def get_config_with_descriptions(
+    schematransform: SchemaTransformsConfig) -> Dict[str, ParamInfo]:
+  # Prepare a configuration schema that includes types and descriptions
+  schema = named_tuple_to_schema(schematransform.configuration_schema)
+  descriptions = schematransform.configuration_schema._field_descriptions
+  fields_with_descriptions = {}
+  for field in schema.fields:
+    fields_with_descriptions[camel_case_to_snake_case(field.name)] = ParamInfo(
+        typing_from_runner_api(field.type),
+        descriptions[field.name],
+        field.name)
+
+  return fields_with_descriptions
+
+
+class ExternalSchemaTransform(PTransform):
+  """Template for a wrapper class of an external SchemaTransform
+
+  This is a superclass for dynamically generated SchemaTransform wrappers and
+  is not meant to be manually instantiated.
+
+  Experimental; no backwards compatibility guarantees."""
+
+  # These attributes need to be set when
+  # creating an ExternalSchemaTransform type
+  default_expansion_service = None
+  description: str = ""
+  identifier: str = ""
+  configuration_schema: Dict[str, ParamInfo] = {}
+
+  def __init__(self, expansion_service=None, **kwargs):
+    self._kwargs = kwargs
+    self._expansion_service = \
+        expansion_service or self.default_expansion_service
+
+  def expand(self, input):
+    camel_case_kwargs = {
+        snake_case_to_lower_camel_case(k): v
+        for k, v in self._kwargs.items()
+    }
+
+    external_schematransform = SchemaAwareExternalTransform(
+        identifier=self.identifier,
+        expansion_service=self._expansion_service,
+        rearrange_based_on_discovery=True,
+        **camel_case_kwargs)
+
+    return input | external_schematransform
+
+
+STANDARD_URN_PATTERN = r"^beam:schematransform:org.apache.beam:([\w-]+):(\w+)$"
+
+
+def infer_name_from_identifier(identifier: str, pattern: str):
+  """Infer a class name from an identifier, adhering to the input pattern"""
+  match = re.match(pattern, identifier)
+  if not match:
+    return None
+  groups = match.groups()
+
+  components = [snake_case_to_upper_camel_case(n) for n in groups]
+  # Special handling for standard SchemaTransform identifiers:
+  # We don't include the version number if it's the first version
+  if (pattern == STANDARD_URN_PATTERN and components[1].lower() == 'v1'):
+    return components[0]
+  else:
+    return ''.join(components)
+
+
+class ExternalSchemaTransformProvider:
+  """Dynamically discovers Schema-aware external transforms from a given list
+  of expansion services and provides them as ready PTransforms.
+
+  A :class:`ExternalSchemaTransform` subclass is generated for each external
+  transform, and is named based on what can be inferred from the URN
+  (see :param urn_pattern).
+
+  These classes are generated when :class:`ExternalSchemaTransformProvider` is
+  initialized. We need to give it one or more expansion service addresses that
+  are already up and running:
+  >>> provider = ExternalSchemaTransformProvider(["localhost:12345",
+  ...                                             "localhost:12121"])
+  We can also give it the gradle target of a standard Beam expansion service:
+  >>> provider = ExternalSchemaTransform(BeamJarExpansionService(
+  ...     "sdks:java:io:google-cloud-platform:expansion-service:shadowJar"))
+  Let's take a look at the output of :func:`get_available()` to know the
+  available transforms in the expansion service(s) we provided:
+  >>> provider.get_available()
+  [('JdbcWrite', 'beam:schematransform:org.apache.beam:jdbc_write:v1'),
+  ('BigtableRead', 'beam:schematransform:org.apache.beam:bigtable_read:v1'),
+  ...]
+
+  Then retrieve a transform by :func:`get()`, :func:`get_urn()`, or by directly
+  accessing it as an attribute of :class:`ExternalSchemaTransformProvider`.
+  All of the following commands do the same thing:
+  >>> provider.get('BigqueryStorageRead')
+  >>> provider.get_urn(
+  ...       'beam:schematransform:org.apache.beam:bigquery_storage_read:v1')
+  >>> provider.BigqueryStorageRead
+
+  To know more about the usage of a given transform, take a look at the
+  `description` attribute. This returns some documentation IF the underlying
+  SchemaTransform provides any.
+  >>> provider.BigqueryStorageRead.description
+
+  Similarly, the `configuration_schema` attribute returns information about the
+  parameters, including their names, types, and any documentation that the
+  underlying SchemaTransform may provide:
+  >>> provider.BigqueryStorageRead.configuration_schema
+  {'query': ParamInfo(type=typing.Optional[str], description='The SQL query to
+  be executed to read from the BigQuery table.', original_name='query'),
+  'row_restriction': ParamInfo(type=typing.Optional[str]...}
+
+  The retrieved external transform can be used as a normal PTransform like so::
+
+    with Pipeline() as p:
+      _ = (p
+        | 'Read from BigQuery` >> provider.BigqueryStorageRead(
+                query=query,
+                row_restriction=restriction)
+        | 'Some processing' >> beam.Map(...))
+
+  Experimental; no backwards compatibility guarantees.
+  """
+  def __init__(self, expansion_services, urn_pattern=STANDARD_URN_PATTERN):
+    f"""Initialize an ExternalSchemaTransformProvider
+
+    :param expansion_services:
+      A list of expansion services to discover transforms from.
+      Supported forms:
+      * a string representing the expansion service address
+      * a :attr:`BeamJarExpansionService` pointing to a gradle target
+    :param urn_pattern:
+      The regular expression used to match valid transforms. In addition to
+      validating, the captured groups are used to infer a name for each class.
+      By default, the following pattern is used: [{STANDARD_URN_PATTERN}]
+    """
+    self._urn_pattern = urn_pattern
+    self._transforms: Dict[str, type(ExternalSchemaTransform)] = {}
+    self._name_to_urn: Dict[str, str] = {}
+
+    if isinstance(expansion_services, set):
+      expansion_services = list(expansion_services)
+    if not isinstance(expansion_services, list):
+      expansion_services = [expansion_services]
+    self.expansion_services = expansion_services
+    self._create_wrappers()
+
+  def _create_wrappers(self):
+    # multiple services can overlap and include the same URNs. If this happens,
+    # we prioritize by the order of services in the list
+    identifiers = set()
+    for service in self.expansion_services:
+      target = service
+      if isinstance(service, BeamJarExpansionService):
+        target = service.gradle_target
+      try:
+        schematransform_configs = SchemaAwareExternalTransform.discover(service)
+      except Exception as e:
+        logging.exception(
+            "Encountered an error while discovering expansion service %s:\n%s",
+            target,
+            e)
+        continue
+      skipped_urns = []
+      for config in schematransform_configs:
+        identifier = config.identifier
+        if identifier not in identifiers:
+          identifiers.add(identifier)
+
+          name = infer_name_from_identifier(identifier, self._urn_pattern)
+          if name is None:
+            skipped_urns.append(identifier)
+            continue
+
+          self._transforms[identifier] = type(
+              name, (ExternalSchemaTransform, ),
+              dict(
+                  identifier=identifier,
+                  default_expansion_service=service,
+                  schematransform=config,
+                  description=config.description,
+                  configuration_schema=get_config_with_descriptions(config)))
+          self._name_to_urn[name] = identifier
+
+      if skipped_urns:
+        logging.info(
+            "Skipped URN(s) in %s that don't follow the pattern [%s]: %s",
+            target,
+            self._urn_pattern,
+            skipped_urns)
+
+    for transform in self._transforms.values():
+      setattr(self, transform.__name__, transform)
+
+  def get_available(self) -> List[Tuple[str, str]]:
+    """Get a list of available ExternalSchemaTransform names and identifiers"""
+    return list(self._name_to_urn.items())
+
+  def get(self, name) -> ExternalSchemaTransform:
+    """Get an ExternalSchemaTransform by its inferred class name"""
+    return self._transforms[self._name_to_urn[name]]
+
+  def get_urn(self, identifier) -> ExternalSchemaTransform:
+    """Get an ExternalSchemaTransform by its SchemaTransform identifier"""
+    return self._transforms[identifier]
diff --git a/sdks/python/apache_beam/transforms/external_schematransform_provider_test.py b/sdks/python/apache_beam/transforms/external_schematransform_provider_test.py
new file mode 100644
index 000000000000..bf951e671c2a
--- /dev/null
+++ b/sdks/python/apache_beam/transforms/external_schematransform_provider_test.py
@@ -0,0 +1,140 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import logging
+import os
+import unittest
+
+import pytest
+
+import apache_beam as beam
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+from apache_beam.transforms.external import BeamJarExpansionService
+from apache_beam.transforms.external_schematransform_provider import STANDARD_URN_PATTERN
+from apache_beam.transforms.external_schematransform_provider import ExternalSchemaTransformProvider
+from apache_beam.transforms.external_schematransform_provider import camel_case_to_snake_case
+from apache_beam.transforms.external_schematransform_provider import infer_name_from_identifier
+from apache_beam.transforms.external_schematransform_provider import snake_case_to_lower_camel_case
+from apache_beam.transforms.external_schematransform_provider import snake_case_to_upper_camel_case
+
+
+class NameUtilsTest(unittest.TestCase):
+  def test_snake_case_to_upper_camel_case(self):
+    test_cases = [("", ""), ("test", "Test"), ("test_name", "TestName"),
+                  ("test_double_underscore", "TestDoubleUnderscore"),
+                  ("TEST_CAPITALIZED", "TestCapitalized"),
+                  ("_prepended_underscore", "PrependedUnderscore"),
+                  ("appended_underscore_", "AppendedUnderscore")]
+    for case in test_cases:
+      self.assertEqual(case[1], snake_case_to_upper_camel_case(case[0]))
+
+  def test_snake_case_to_lower_camel_case(self):
+    test_cases = [("", ""), ("test", "test"), ("test_name", "testName"),
+                  ("test_double_underscore", "testDoubleUnderscore"),
+                  ("TEST_CAPITALIZED", "testCapitalized"),
+                  ("_prepended_underscore", "prependedUnderscore"),
+                  ("appended_underscore_", "appendedUnderscore")]
+    for case in test_cases:
+      self.assertEqual(case[1], snake_case_to_lower_camel_case(case[0]))
+
+  def test_camel_case_to_snake_case(self):
+    test_cases = [("", ""), ("Test", "test"), ("TestName", "test_name"),
+                  ("TestDoubleUnderscore",
+                   "test_double_underscore"), ("MyToLoFo", "my_to_lo_fo"),
+                  ("BEGINNINGAllCaps",
+                   "beginning_all_caps"), ("AllCapsENDING", "all_caps_ending"),
+                  ("AllCapsMIDDLEWord", "all_caps_middle_word"),
+                  ("lowerCamelCase", "lower_camel_case")]
+    for case in test_cases:
+      self.assertEqual(case[1], camel_case_to_snake_case(case[0]))
+
+  def test_infer_name_from_identifier(self):
+    standard_test_cases = [
+        ("beam:schematransform:org.apache.beam:transform:v1", "Transform"),
+        ("beam:schematransform:org.apache.beam:my_transform:v1",
+         "MyTransform"), (
+             "beam:schematransform:org.apache.beam:my_transform:v2",
+             "MyTransformV2"),
+        ("beam:schematransform:org.apache.beam:fe_fi_fo_fum:v2", "FeFiFoFumV2"),
+        ("beam:schematransform:bad_match:my_transform:v1", None)
+    ]
+    for case in standard_test_cases:
+      self.assertEqual(
+          case[1], infer_name_from_identifier(case[0], STANDARD_URN_PATTERN))
+
+    custom_pattern_cases = [
+        # (<pattern>, <urn>, <expected output>)
+        (
+            r"^custom:transform:([\w-]+):(\w+)$",
+            "custom:transform:my_transform:v1",
+            "MyTransformV1"),
+        (
+            r"^org.user:([\w-]+):([\w-]+):([\w-]+):external$",
+            "org.user:some:custom_transform:we_made:external",
+            "SomeCustomTransformWeMade"),
+        (
+            r"^([\w-]+):user.transforms",
+            "my_eXTErnal:user.transforms",
+            "MyExternal"),
+        (r"^([\w-]+):user.transforms", "my_external:badinput.transforms", None),
+    ]
+    for case in custom_pattern_cases:
+      self.assertEqual(case[2], infer_name_from_identifier(case[1], case[0]))
+
+
+@pytest.mark.uses_io_java_expansion_service
+@unittest.skipUnless(
+    os.environ.get('EXPANSION_PORT'),
+    "EXPANSION_PORT environment var is not provided.")
+class ExternalSchemaTransformProviderTest(unittest.TestCase):
+  def setUp(self):
+    self.test_pipeline = TestPipeline(is_integration_test=True)
+
+  def test_generate_sequence_config_schema_and_description(self):
+    provider = ExternalSchemaTransformProvider(
+        BeamJarExpansionService(":sdks:java:io:expansion-service:shadowJar"))
+
+    self.assertTrue((
+        'GenerateSequence',
+        'beam:schematransform:org.apache.beam:generate_sequence:v1'
+    ) in provider.get_available())
+
+    GenerateSequence = provider.get('GenerateSequence')
+    config_schema = GenerateSequence.configuration_schema
+    for param in ['start', 'end', 'rate']:
+      self.assertTrue(param in config_schema)
+
+    description_substring = (
+        "Outputs a PCollection of Beam Rows, each "
+        "containing a single INT64")
+    self.assertTrue(description_substring in GenerateSequence.description)
+
+  def test_run_generate_sequence(self):
+    provider = ExternalSchemaTransformProvider(
+        BeamJarExpansionService(":sdks:java:io:expansion-service:shadowJar"))
+
+    with beam.Pipeline() as p:
+      numbers = p | provider.GenerateSequence(
+          start=0, end=10) | beam.Map(lambda row: row.value)
+
+      assert_that(numbers, equal_to([i for i in range(10)]))
+
+
+if __name__ == '__main__':
+  logging.getLogger().setLevel(logging.INFO)
+  unittest.main()
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index fcff86d4c50c..c99d90e184e2 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -37,6 +37,8 @@ class and wrapper class that allows lambda functions to be used as
 # pytype: skip-file
 
 import copy
+import functools
+import inspect
 import itertools
 import json
 import logging
@@ -1076,6 +1078,15 @@ def callable_ptransform_factory(*args, **kwargs):
           'type hints for %s: %s', res.default_label(), res.get_type_hints())
     return res
 
+  # The signature of this PTransform constructor is that of fn minus the first
+  # argument (which is where the pvalue is passed during expand).
+  try:
+    callable_ptransform_factory.__signature__ = inspect.signature(  # type: ignore
+        functools.partial(fn, None))
+  except Exception:
+    # Sometimes we can't get the original signature.
+    pass
+
   return callable_ptransform_factory
 
 
diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py
index cacfdb37d7b4..c554bef6c36d 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -70,6 +70,7 @@
 from apache_beam.typehints import trivial_inference
 from apache_beam.typehints.decorators import get_signature
 from apache_beam.typehints.sharded_key_type import ShardedKeyType
+from apache_beam.utils import shared
 from apache_beam.utils import windowed_value
 from apache_beam.utils.annotations import deprecated
 from apache_beam.utils.sharded_key import ShardedKey
@@ -748,6 +749,32 @@ def flush_batch(
   return _StatefulBatchElementsDoFn()
 
 
+class SharedKey():
+  """A class that holds a per-process UUID used to key elements for streaming
+  BatchElements. 
+  """
+  def __init__(self):
+    self.key = uuid.uuid4().hex
+
+
+def load_shared_key():
+  return SharedKey()
+
+
+class WithSharedKey(DoFn):
+  """A DoFn that keys elements with a per-process UUID. Used in streaming
+  BatchElements.  
+  """
+  def __init__(self):
+    self.shared_handle = shared.Shared()
+
+  def setup(self):
+    self.key = self.shared_handle.acquire(load_shared_key, "WithSharedKey").key
+
+  def process(self, element):
+    yield (self.key, element)
+
+
 @typehints.with_input_types(T)
 @typehints.with_output_types(List[T])
 class BatchElements(PTransform):
@@ -826,7 +853,7 @@ def expand(self, pcoll):
       raise NotImplementedError("Requires stateful processing (BEAM-2687)")
     elif self._max_batch_dur is not None:
       coder = coders.registry.get_coder(pcoll)
-      return pcoll | WithKeys(0) | ParDo(
+      return pcoll | ParDo(WithSharedKey()) | ParDo(
           _pardo_stateful_batch_elements(
               coder,
               self._batch_size_estimator,
diff --git a/sdks/python/apache_beam/typehints/schemas.py b/sdks/python/apache_beam/typehints/schemas.py
index ea836430e8e2..147a46f0bea5 100644
--- a/sdks/python/apache_beam/typehints/schemas.py
+++ b/sdks/python/apache_beam/typehints/schemas.py
@@ -245,6 +245,10 @@ def typing_to_runner_api(self, type_: type) -> schema_pb2.FieldType:
     if isinstance(type_, schema_pb2.Schema):
       return schema_pb2.FieldType(row_type=schema_pb2.RowType(schema=type_))
 
+    if hasattr(type_, '_beam_schema_proto') and type_._beam_schema_proto.obj:
+      return schema_pb2.FieldType(
+          row_type=schema_pb2.RowType(schema=type_._beam_schema_proto.obj))
+
     if isinstance(type_, row_type.RowTypeConstraint):
       if type_.schema_id is None:
         schema_id = SCHEMA_REGISTRY.generate_new_id()
@@ -536,6 +540,7 @@ def named_tuple_from_schema(self, schema: schema_pb2.Schema) -> type:
     type_name = 'BeamSchema_{}'.format(schema.id.replace('-', '_'))
 
     subfields = []
+    descriptions = {}
     for field in schema.fields:
       try:
         field_py_type = self.typing_from_runner_api(field.type)
@@ -546,6 +551,7 @@ def named_tuple_from_schema(self, schema: schema_pb2.Schema) -> type:
             "Failed to decode schema due to an issue with Field proto:\n\n"
             f"{text_format.MessageToString(field)}") from e
 
+      descriptions[field.name] = field.description
       subfields.append((field.name, field_py_type))
 
     user_type = NamedTuple(type_name, subfields)
@@ -556,6 +562,7 @@ def named_tuple_from_schema(self, schema: schema_pb2.Schema) -> type:
         user_type,
         '__reduce__',
         _named_tuple_reduce_method(schema.SerializeToString()))
+    setattr(user_type, "_field_descriptions", descriptions)
     setattr(user_type, row_type._BEAM_SCHEMA_ID, schema.id)
 
     self.schema_registry.add(user_type, schema)
@@ -627,6 +634,15 @@ def union_schema_type(element_types):
   return named_tuple_from_schema(named_fields_to_schema(union_fields_and_types))
 
 
+class _Ephemeral:
+  """Helper class for wrapping unpicklable objects."""
+  def __init__(self, obj):
+    self.obj = obj
+
+  def __reduce__(self):
+    return _Ephemeral, (None, )
+
+
 # Registry of typings for a schema by UUID
 class LogicalTypeRegistry(object):
   def __init__(self):
diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py
index 4fd4b97e82cd..70eb78b6ffc6 100644
--- a/sdks/python/apache_beam/typehints/typehints.py
+++ b/sdks/python/apache_beam/typehints/typehints.py
@@ -1020,13 +1020,13 @@ def __getitem__(self, type_param):
 
 class CollectionHint(CompositeTypeHint):
   """ A Collection type-hint.
-  
+
   Collection[X] defines a type-hint for a collection of homogenous types. 'X'
   may be either a built-in Python type or another nested TypeConstraint.
 
   This represents a collections.abc.Collection type, which implements
   __contains__, __iter__, and __len__. This acts as a parent type for
-  sets but has fewer guarantees for mixins. 
+  sets but has fewer guarantees for mixins.
   """
   class CollectionTypeConstraint(SequenceTypeConstraint):
     def __init__(self, type_param):
@@ -1302,6 +1302,8 @@ def is_consistent_with(sub, base):
   relation, but also handles the special Any type as well as type
   parameterization.
   """
+  from apache_beam.pvalue import Row
+  from apache_beam.typehints.row_type import RowTypeConstraint
   if sub == base:
     # Common special case.
     return True
@@ -1313,6 +1315,8 @@ def is_consistent_with(sub, base):
     return all(is_consistent_with(c, base) for c in sub.union_types)
   elif isinstance(base, TypeConstraint):
     return base._consistent_with_check_(sub)
+  elif isinstance(sub, RowTypeConstraint):
+    return base == Row
   elif isinstance(sub, TypeConstraint):
     # Nothing but object lives above any type constraints.
     return base == object
diff --git a/sdks/python/apache_beam/utils/retry.py b/sdks/python/apache_beam/utils/retry.py
index 6eed2900b9a4..485fc9d627e9 100644
--- a/sdks/python/apache_beam/utils/retry.py
+++ b/sdks/python/apache_beam/utils/retry.py
@@ -28,6 +28,7 @@
 # pytype: skip-file
 
 import functools
+import json
 import logging
 import random
 import sys
@@ -57,6 +58,7 @@
 # pylint: enable=wrong-import-order, wrong-import-position
 
 _LOGGER = logging.getLogger(__name__)
+_RETRYABLE_REASONS = ["rateLimitExceeded", "internalError", "backendError"]
 
 
 class PermanentException(Exception):
@@ -166,17 +168,38 @@ def retry_on_server_errors_and_timeout_filter(exception):
 
 
 def retry_on_server_errors_timeout_or_quota_issues_filter(exception):
-  """Retry on server, timeout and 403 errors.
+  """Retry on server, timeout, 429, and some 403 errors.
 
-  403 errors can be accessDenied, billingNotEnabled, and also quotaExceeded,
-  rateLimitExceeded."""
+  403 errors from BigQuery include both non-transient (accessDenied,
+  billingNotEnabled) and transient errors (rateLimitExceeded).
+  Only retry transient errors."""
   if HttpError is not None and isinstance(exception, HttpError):
-    if exception.status_code == 403:
+    if exception.status_code == 429:
       return True
+    if exception.status_code == 403:
+      try:
+        # attempt to extract the reason and check if it's retryable
+        content = exception.content
+        if not isinstance(content, dict):
+          content = json.loads(exception.content)
+        return content["error"]["errors"][0]["reason"] in _RETRYABLE_REASONS
+      except (KeyError, IndexError, TypeError) as e:
+        _LOGGER.warning(
+            "Could not determine if HttpError is transient. "
+            "Will not retry: %s",
+            e)
+      return False
   if GoogleAPICallError is not None and isinstance(exception,
                                                    GoogleAPICallError):
-    if exception.code == 403:
+    if exception.code == 429:
       return True
+    if exception.code == 403:
+      if not hasattr(exception, "errors") or len(exception.errors) == 0:
+        # default to not retrying
+        return False
+
+      reason = exception.errors[0]["reason"]
+      return reason in _RETRYABLE_REASONS
   if S3ClientError is not None and isinstance(exception, S3ClientError):
     if exception.code == 403:
       return True
diff --git a/sdks/python/apache_beam/utils/subprocess_server.py b/sdks/python/apache_beam/utils/subprocess_server.py
index 6ed1568b57a5..7d9cf50d8532 100644
--- a/sdks/python/apache_beam/utils/subprocess_server.py
+++ b/sdks/python/apache_beam/utils/subprocess_server.py
@@ -18,6 +18,7 @@
 # pytype: skip-file
 
 import contextlib
+import dataclasses
 import glob
 import hashlib
 import logging
@@ -27,10 +28,11 @@
 import signal
 import socket
 import subprocess
-import tempfile
 import threading
 import time
 import zipfile
+from typing import Any
+from typing import Set
 from urllib.error import URLError
 from urllib.request import urlopen
 
@@ -42,6 +44,75 @@
 _LOGGER = logging.getLogger(__name__)
 
 
+@dataclasses.dataclass
+class _SharedCacheEntry:
+  obj: Any
+  owners: Set[str]
+
+
+class _SharedCache:
+  """A cache that keeps objects alive (and repeatedly returns the same instance)
+  until the last user indicates that they're done.
+
+  The typical usage is as follows::
+
+    try:
+      token = cache.register()
+      # All objects retrieved from the cache from this point on will be memoized
+      # and kept alive (including across other threads and callers) at least
+      # until the purge is called below (and possibly longer, if other calls
+      # to register were made).
+      obj = cache.get(...)
+      another_obj = cache.get(...)
+      ...
+    finally:
+      cache.purge(token)
+  """
+  def __init__(self, constructor, destructor):
+    self._constructor = constructor
+    self._destructor = destructor
+    self._live_owners = set()
+    self._cache = {}
+    self._lock = threading.Lock()
+    self._counter = 0
+
+  def _next_id(self):
+    with self._lock:
+      self._counter += 1
+      return self._counter
+
+  def register(self):
+    owner = self._next_id()
+    self._live_owners.add(owner)
+    return owner
+
+  def purge(self, owner):
+    if owner not in self._live_owners:
+      raise ValueError(f"{owner} not in {self._live_owners}")
+    self._live_owners.remove(owner)
+    to_delete = []
+    with self._lock:
+      for key, entry in list(self._cache.items()):
+        if owner in entry.owners:
+          entry.owners.remove(owner)
+        if not entry.owners:
+          to_delete.append(entry.obj)
+          del self._cache[key]
+    # Actually call the destructors outside of the lock.
+    for value in to_delete:
+      self._destructor(value)
+
+  def get(self, *key):
+    if not self._live_owners:
+      raise RuntimeError("At least one owner must be registered.")
+    with self._lock:
+      if key not in self._cache:
+        self._cache[key] = _SharedCacheEntry(self._constructor(*key), set())
+      for owner in self._live_owners:
+        self._cache[key].owners.add(owner)
+      return self._cache[key].obj
+
+
 class SubprocessServer(object):
   """An abstract base class for running GRPC Servers as an external process.
 
@@ -63,13 +134,26 @@ def __init__(self, stub_class, cmd, port=None):
         string "{{PORT}}" will be substituted in the command line arguments
         with the chosen port.
     """
-    self._process_lock = threading.RLock()
-    self._process = None
+    self._owner_id = None
     self._stub_class = stub_class
     self._cmd = [str(arg) for arg in cmd]
     self._port = port
     self._grpc_channel = None
 
+  @classmethod
+  @contextlib.contextmanager
+  def cache_subprocesses(cls):
+    """A context that ensures any subprocess created or used in its duration
+    stay alive for at least the duration of this context.
+
+    These subprocesses may be shared with other contexts as well.
+    """
+    try:
+      unique_id = cls._cache.register()
+      yield
+    finally:
+      cls._cache.purge(unique_id)
+
   def __enter__(self):
     return self.start()
 
@@ -78,7 +162,7 @@ def __exit__(self, *unused_args):
 
   def start(self):
     try:
-      endpoint = self.start_process()
+      process, endpoint = self.start_process()
       wait_secs = .1
       channel_options = [("grpc.max_receive_message_length", -1),
                          ("grpc.max_send_message_length", -1)]
@@ -86,10 +170,10 @@ def start(self):
           endpoint, options=channel_options)
       channel_ready = grpc.channel_ready_future(self._grpc_channel)
       while True:
-        if self._process is not None and self._process.poll() is not None:
-          _LOGGER.error("Starting job service with %s", self._process.args)
+        if process is not None and process.poll() is not None:
+          _LOGGER.error("Started job service with %s", process.args)
           raise RuntimeError(
-              'Service failed to start up with error %s' % self._process.poll())
+              'Service failed to start up with error %s' % process.poll())
         try:
           channel_ready.result(timeout=wait_secs)
           break
@@ -106,60 +190,66 @@ def start(self):
       raise
 
   def start_process(self):
-    with self._process_lock:
-      if self._process:
-        self.stop()
-      if self._port:
-        port = self._port
-        cmd = self._cmd
-      else:
-        port, = pick_port(None)
-        cmd = [arg.replace('{{PORT}}', str(port)) for arg in self._cmd]
-      endpoint = 'localhost:%s' % port
-      _LOGGER.info("Starting service with %s", str(cmd).replace("',", "'"))
-      self._process = subprocess.Popen(
-          cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-      # Emit the output of this command as info level logging.
-      def log_stdout():
-        line = self._process.stdout.readline()
-        while line:
-          # The log obtained from stdout is bytes, decode it into string.
-          # Remove newline via rstrip() to not print an empty line.
-          _LOGGER.info(line.decode(errors='backslashreplace').rstrip())
-          line = self._process.stdout.readline()
-
-      t = threading.Thread(target=log_stdout)
-      t.daemon = True
-      t.start()
-      return endpoint
+    if self._owner_id is not None:
+      self._cache.purge(self._owner_id)
+    self._owner_id = self._cache.register()
+    return self._cache.get(tuple(self._cmd), self._port)
+
+  def _really_start_process(cmd, port):
+    if not port:
+      port, = pick_port(None)
+      cmd = [arg.replace('{{PORT}}', str(port)) for arg in cmd]  # pylint: disable=not-an-iterable
+    endpoint = 'localhost:%s' % port
+    _LOGGER.info("Starting service with %s", str(cmd).replace("',", "'"))
+    process = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+    # Emit the output of this command as info level logging.
+    def log_stdout():
+      line = process.stdout.readline()
+      while line:
+        # The log obtained from stdout is bytes, decode it into string.
+        # Remove newline via rstrip() to not print an empty line.
+        _LOGGER.info(line.decode(errors='backslashreplace').rstrip())
+        line = process.stdout.readline()
+
+    t = threading.Thread(target=log_stdout)
+    t.daemon = True
+    t.start()
+    return process, endpoint
 
   def stop(self):
     self.stop_process()
 
   def stop_process(self):
-    with self._process_lock:
-      if not self._process:
-        return
-      for _ in range(5):
-        if self._process.poll() is not None:
-          break
-        logging.debug("Sending SIGINT to job_server")
-        self._process.send_signal(signal.SIGINT)
-        time.sleep(1)
-      if self._process.poll() is None:
-        self._process.kill()
-      self._process = None
+    if self._owner_id is not None:
+      self._cache.purge(self._owner_id)
+      self._owner_id = None
     if self._grpc_channel:
       try:
         self._grpc_channel.close()
       except:  # pylint: disable=bare-except
         _LOGGER.error(
-            "Could not close the gRPC channel started for the " +
+            "Could not close the gRPC channel started for the "
             "expansion service")
-
-  def local_temp_dir(self, **kwargs):
-    return tempfile.mkdtemp(dir=self._local_temp_root, **kwargs)
+      finally:
+        self._grpc_channel = None
+
+  def _really_stop_process(process_and_endpoint):
+    process, _ = process_and_endpoint  # pylint: disable=unpacking-non-sequence
+    if not process:
+      return
+    for _ in range(5):
+      if process.poll() is not None:
+        break
+      logging.debug("Sending SIGINT to process")
+      process.send_signal(signal.SIGINT)
+      time.sleep(1)
+    if process.poll() is None:
+      process.kill()
+
+  _cache = _SharedCache(
+      constructor=_really_start_process, destructor=_really_stop_process)
 
 
 class JavaJarServer(SubprocessServer):
@@ -184,7 +274,7 @@ def __init__(self, stub_class, path_to_jar, java_arguments, classpath=None):
 
   def start_process(self):
     if self._existing_service:
-      return self._existing_service
+      return None, self._existing_service
     else:
       if not shutil.which('java'):
         raise RuntimeError(
diff --git a/sdks/python/apache_beam/utils/subprocess_server_test.py b/sdks/python/apache_beam/utils/subprocess_server_test.py
index e0d0892c8e68..c0c8e5694b86 100644
--- a/sdks/python/apache_beam/utils/subprocess_server_test.py
+++ b/sdks/python/apache_beam/utils/subprocess_server_test.py
@@ -21,6 +21,7 @@
 
 import glob
 import os
+import random
 import re
 import shutil
 import socketserver
@@ -166,5 +167,83 @@ def test_classpath_jar(self):
         os.chdir(oldwd)
 
 
+class CacheTest(unittest.TestCase):
+  @staticmethod
+  def with_prefix(prefix):
+    return '%s-%s' % (prefix, random.random())
+
+  def test_memoization(self):
+    cache = subprocess_server._SharedCache(self.with_prefix, lambda x: None)
+    try:
+      token = cache.register()
+      a = cache.get('a')
+      self.assertEqual(a[0], 'a')
+      self.assertEqual(cache.get('a'), a)
+      b = cache.get('b')
+      self.assertEqual(b[0], 'b')
+      self.assertEqual(cache.get('b'), b)
+    finally:
+      cache.purge(token)
+
+  def test_purged(self):
+    cache = subprocess_server._SharedCache(self.with_prefix, lambda x: None)
+    try:
+      token = cache.register()
+      a = cache.get('a')
+      self.assertEqual(cache.get('a'), a)
+    finally:
+      cache.purge(token)
+
+    try:
+      token = cache.register()
+      new_a = cache.get('a')
+      self.assertNotEqual(new_a, a)
+    finally:
+      cache.purge(token)
+
+  def test_multiple_owners(self):
+    cache = subprocess_server._SharedCache(self.with_prefix, lambda x: None)
+    try:
+      owner1 = cache.register()
+      a = cache.get('a')
+      try:
+        self.assertEqual(cache.get('a'), a)
+        owner2 = cache.register()
+        b = cache.get('b')
+        self.assertEqual(cache.get('b'), b)
+      finally:
+        cache.purge(owner2)
+      self.assertEqual(cache.get('a'), a)
+      self.assertEqual(cache.get('b'), b)
+    finally:
+      cache.purge(owner1)
+
+    try:
+      owner3 = cache.register()
+      self.assertNotEqual(cache.get('a'), a)
+      self.assertNotEqual(cache.get('b'), b)
+    finally:
+      cache.purge(owner3)
+
+  def test_interleaved_owners(self):
+    cache = subprocess_server._SharedCache(self.with_prefix, lambda x: None)
+    owner1 = cache.register()
+    a = cache.get('a')
+    self.assertEqual(cache.get('a'), a)
+
+    owner2 = cache.register()
+    b = cache.get('b')
+    self.assertEqual(cache.get('b'), b)
+
+    cache.purge(owner1)
+    self.assertNotEqual(cache.get('a'), a)
+    self.assertEqual(cache.get('b'), b)
+
+    cache.purge(owner2)
+    owner3 = cache.register()
+    self.assertNotEqual(cache.get('b'), b)
+    cache.purge(owner3)
+
+
 if __name__ == '__main__':
   unittest.main()
diff --git a/sdks/python/apache_beam/version.py b/sdks/python/apache_beam/version.py
index fa890eab5005..f0df92e31e57 100644
--- a/sdks/python/apache_beam/version.py
+++ b/sdks/python/apache_beam/version.py
@@ -17,4 +17,4 @@
 
 """Apache Beam SDK version information and utilities."""
 
-__version__ = '2.53.0.dev'
+__version__ = '2.54.0.dev'
diff --git a/sdks/python/apache_beam/yaml/README.md b/sdks/python/apache_beam/yaml/README.md
index 247b42b6839a..395cd5321967 100644
--- a/sdks/python/apache_beam/yaml/README.md
+++ b/sdks/python/apache_beam/yaml/README.md
@@ -44,8 +44,9 @@ or consumption (e.g. a lineage analysis tool) and expect it to be more
 easily manipulated and semantically meaningful than the Beam protos
 themselves (which concern themselves more with execution).
 
-It should be noted that everything here is still EXPERIMENTAL and subject
-to change. Feedback is welcome at dev@apache.beam.org.
+It should be noted that everything here is still under development, but any
+features already included are considered stable. Feedback is welcome at
+dev@apache.beam.org.
 
 ## Example pipelines
 
@@ -273,6 +274,13 @@ pipeline:
     - type: ReadFromPubSub
       config:
         topic: myPubSubTopic
+        format: json
+        schema:
+          type: object
+          properties:
+            col1: {type: string}
+            col2: {type: integer}
+            col3: {type: number}
     - type: WindowInto
       windowing:
         type: fixed
@@ -281,6 +289,7 @@ pipeline:
     - type: WriteToPubSub
       config:
         topic: anotherPubSubTopic
+        format: json
 ```
 
 Rather than using an explicit `WindowInto` operation, one may instead tag a
@@ -294,6 +303,8 @@ pipeline:
     - type: ReadFromPubSub
       config:
         topic: myPubSubTopic
+        format: ...
+        schema: ...
     - type: SomeAggregation
       windowing:
         type: sliding
@@ -302,6 +313,7 @@ pipeline:
     - type: WriteToPubSub
       config:
         topic: anotherPubSubTopic
+        format: json
 ```
 
 Note that the `Sql` operation itself is often a from of aggregation, and
@@ -315,6 +327,8 @@ pipeline:
     - type: ReadFromPubSub
       config:
         topic: myPubSubTopic
+        format: ...
+        schema: ...
     - type: Sql
       config:
         query: "select col1, count(*) as c from PCOLLECTION"
@@ -324,6 +338,7 @@ pipeline:
     - type: WriteToPubSub
       config:
         topic: anotherPubSubTopic
+        format: json
 ```
 
 The specified windowing is applied to all inputs, in this case resulting in
@@ -336,11 +351,15 @@ pipeline:
       name: ReadLeft
       config:
         topic: leftTopic
+        format: ...
+        schema: ...
 
     - type: ReadFromPubSub
       name: ReadRight
       config:
         topic: rightTopic
+        format: ...
+        schema: ...
 
     - type: Sql
       config:
@@ -363,7 +382,9 @@ pipeline:
   transforms:
     - type: ReadFromPubSub
       config:
-       topic: myPubSubTopic
+        topic: myPubSubTopic
+        format: ...
+        schema: ...
       windowing:
         type: fixed
         size: 60
@@ -373,6 +394,7 @@ pipeline:
     - type: WriteToPubSub
       config:
         topic: anotherPubSubTopic
+        format: json
 ```
 
 One can also specify windowing at the top level of a pipeline (or composite),
@@ -387,12 +409,15 @@ pipeline:
     - type: ReadFromPubSub
       config:
         topic: myPubSubTopic
+        format: ...
+        schema: ...
     - type: Sql
       config:
         query: "select col1, count(*) as c from PCOLLECTION"
     - type: WriteToPubSub
       config:
         topic: anotherPubSubTopic
+        format: json
   windowing:
     type: fixed
     size: 60
@@ -409,6 +434,8 @@ pipeline:
     type: ReadFromPubSub
     config:
       topic: myPubSubTopic
+      format: ...
+      schema: ...
     windowing:
       type: fixed
       size: 10
diff --git a/sdks/python/apache_beam/yaml/generate_yaml_docs.py b/sdks/python/apache_beam/yaml/generate_yaml_docs.py
new file mode 100644
index 000000000000..6b0484171878
--- /dev/null
+++ b/sdks/python/apache_beam/yaml/generate_yaml_docs.py
@@ -0,0 +1,188 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import re
+
+import yaml
+
+from apache_beam.portability.api import schema_pb2
+from apache_beam.utils import subprocess_server
+from apache_beam.yaml import yaml_provider
+
+
+def _fake_value(name, beam_type):
+  type_info = beam_type.WhichOneof("type_info")
+  if type_info == "atomic_type":
+    if beam_type.atomic_type == schema_pb2.STRING:
+      return f'"{name}"'
+    elif beam_type.atomic_type == schema_pb2.BOOLEAN:
+      return "true|false"
+    else:
+      return name
+  elif type_info == "array_type":
+    return [_fake_value(name, beam_type.array_type.element_type), '...']
+  elif type_info == "iterable_type":
+    return [_fake_value(name, beam_type.iterable_type.element_type), '...']
+  elif type_info == "map_type":
+    if beam_type.map_type.key_type.atomic_type == schema_pb2.STRING:
+      return {
+          'a': _fake_value(name + '_value_a', beam_type.map_type.value_type),
+          'b': _fake_value(name + '_value_b', beam_type.map_type.value_type),
+          'c': '...',
+      }
+    else:
+      return {
+          _fake_value(name + '_key', beam_type.map_type.key_type): _fake_value(
+              name + '_value', beam_type.map_type.value_type)
+      }
+  elif type_info == "row_type":
+    return _fake_row(beam_type.row_type.schema)
+  elif type_info == "logical_type":
+    return name
+  else:
+    raise ValueError(f"Unrecognized type_info: {type_info!r}")
+
+
+def _fake_row(schema):
+  if schema is None:
+    return '...'
+  return {f.name: _fake_value(f.name, f.type) for f in schema.fields}
+
+
+def pretty_example(provider, t):
+  spec = {'type': t}
+  try:
+    requires_inputs = provider.requires_inputs(t, {})
+  except Exception:
+    requires_inputs = False
+  if requires_inputs:
+    spec['input'] = '...'
+  config_schema = provider.config_schema(t)
+  if config_schema is None or config_schema.fields:
+    spec['config'] = _fake_row(config_schema)
+  s = yaml.dump(spec, sort_keys=False)
+  return s.replace("'", "")
+
+
+def config_docs(schema):
+  if schema is None:
+    return ''
+  elif not schema.fields:
+    return 'No configuration parameters.'
+
+  def pretty_type(beam_type):
+    type_info = beam_type.WhichOneof("type_info")
+    if type_info == "atomic_type":
+      return schema_pb2.AtomicType.Name(beam_type.atomic_type).lower()
+    elif type_info == "array_type":
+      return f'Array[{pretty_type(beam_type.array_type.element_type)}]'
+    elif type_info == "iterable_type":
+      return f'Iterable[{pretty_type(beam_type.iterable_type.element_type)}]'
+    elif type_info == "map_type":
+      return (
+          f'Map[{pretty_type(beam_type.map_type.key_type)}, '
+          f'{pretty_type(beam_type.map_type.value_type)}]')
+    elif type_info == "row_type":
+      return 'Row'
+    else:
+      return '?'
+
+  def maybe_row_parameters(t):
+    if t.WhichOneof("type_info") == "row_type":
+      return indent('\n\nRow fields:\n\n' + config_docs(t.row_type.schema), 4)
+    else:
+      return ''
+
+  def maybe_optional(t):
+    return " (Optional)" if t.nullable else ""
+
+  def lines():
+    for f in schema.fields:
+      yield ''.join([
+          f'**{f.name}** `{pretty_type(f.type)}`',
+          maybe_optional(f.type),
+          indent(': ' + f.description if f.description else '', 2),
+          maybe_row_parameters(f.type),
+      ])
+
+  return '\n\n'.join('*' + indent(line, 2) for line in lines()).strip()
+
+
+def indent(lines, size):
+  return '\n'.join(' ' * size + line for line in lines.split('\n'))
+
+
+def longest(func, xs):
+  return max([func(x) or '' for x in xs], key=len)
+
+
+def io_grouping_key(transform_name):
+  """Place reads and writes next to each other, after all other transforms."""
+  if transform_name.startswith('ReadFrom'):
+    return 1, transform_name[8:], 0
+  elif transform_name.startswith('WriteTo'):
+    return 1, transform_name[7:], 1
+  else:
+    return 0, transform_name
+
+
+SKIP = [
+    'Combine',
+    'Filter',
+    'MapToFields',
+]
+
+
+def transform_docs(t, providers):
+  return '\n'.join([
+      f'## {t}',
+      '',
+      longest(lambda p: p.description(t), providers),
+      '',
+      '### Configuration',
+      '',
+      longest(lambda p: config_docs(p.config_schema(t)), providers),
+      '',
+      '### Usage',
+      '',
+      indent(longest(lambda p: pretty_example(p, t), providers), 4),
+  ])
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('output_file')
+  parser.add_argument('--include', default='.*')
+  parser.add_argument(
+      '--exclude', default='(Combine)|(Filter)|(MapToFields)-.*')
+  options = parser.parse_args()
+  include = re.compile(options.include).match
+  exclude = re.compile(options.exclude).match
+
+  with subprocess_server.SubprocessServer.cache_subprocesses():
+    with open(options.output_file, 'w') as fout:
+      providers = yaml_provider.standard_providers()
+      for transform in sorted(providers.keys(), key=io_grouping_key):
+        if include(transform) and not exclude(transform):
+          print(transform)
+          fout.write(transform_docs(transform, providers[transform]))
+          fout.write('\n\n')
+
+
+if __name__ == '__main__':
+  main()
diff --git a/sdks/python/apache_beam/yaml/readme_test.py b/sdks/python/apache_beam/yaml/readme_test.py
index 7f2d193bf35f..8ec9b22cc92f 100644
--- a/sdks/python/apache_beam/yaml/readme_test.py
+++ b/sdks/python/apache_beam/yaml/readme_test.py
@@ -99,7 +99,7 @@ def guess_name_and_type(expr):
 
 
 class FakeReadFromPubSub(beam.PTransform):
-  def __init__(self, topic):
+  def __init__(self, topic, format, schema):
     pass
 
   def expand(self, p):
@@ -112,7 +112,7 @@ def expand(self, p):
 
 
 class FakeWriteToPubSub(beam.PTransform):
-  def __init__(self, topic):
+  def __init__(self, topic, format):
     pass
 
   def expand(self, pcoll):
diff --git a/sdks/python/apache_beam/yaml/standard_io.yaml b/sdks/python/apache_beam/yaml/standard_io.yaml
index b19c1e5b063e..b49d40d5a40b 100644
--- a/sdks/python/apache_beam/yaml/standard_io.yaml
+++ b/sdks/python/apache_beam/yaml/standard_io.yaml
@@ -68,8 +68,7 @@
         'format': 'format'
         'topic': 'topic'
         'bootstrap_servers': 'bootstrapServers'
-        'producer_config_updates': 'ProducerConfigUpdates'
-        'error_handling': 'errorHandling'
+        'producer_config_updates': 'producerConfigUpdates'
         'file_descriptor_path': 'fileDescriptorPath'
         'message_name': 'messageName'
     underlying_provider:
@@ -80,6 +79,36 @@
       config:
         gradle_target: 'sdks:java:io:expansion-service:shadowJar'
 
+- type: renaming
+  transforms:
+    'ReadFromPubSubLite': 'ReadFromPubSubLite'
+    'WriteToPubSubLite': 'WriteToPubSubLite'
+  config:
+    mappings:
+      'ReadFromPubSubLite':
+        'project': 'project'
+        'schema': 'schema'
+        'format': 'format'
+        'subscription_name': 'subscriptionName'
+        'location': 'location'
+        'attributes': 'attributes'
+        'attribute_map': 'attributeMap'
+        'attribute_id': 'attributeId'
+      'WriteToPubSubLite':
+        'project': 'project'
+        'format': 'format'
+        'topic_name': 'topicName'
+        'location': 'location'
+        'attributes': 'attributes'
+        'attribute_id': 'attributeId'
+    underlying_provider:
+      type: beamJar
+      transforms:
+        'ReadFromPubSubLite': 'beam:schematransform:org.apache.beam:pubsublite_read:v1'
+        'WriteToPubSubLite': 'beam:schematransform:org.apache.beam:pubsublite_write:v1'
+      config:
+        gradle_target: 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar'
+
 - type: python
   transforms:
     'ReadFromBigQuery': 'apache_beam.yaml.yaml_io.read_from_bigquery'
@@ -210,13 +239,5 @@
       transforms:
         'ReadFromJdbc': 'beam:schematransform:org.apache.beam:jdbc_read:v1'
         'WriteToJdbc': 'beam:schematransform:org.apache.beam:jdbc_write:v1'
-        'ReadFromMySql': 'beam:schematransform:org.apache.beam:jdbc_read:v1'
-        'WriteToMySql': 'beam:schematransform:org.apache.beam:jdbc_write:v1'
-        'ReadFromPostgres': 'beam:schematransform:org.apache.beam:jdbc_read:v1'
-        'WriteToPostgres': 'beam:schematransform:org.apache.beam:jdbc_write:v1'
-        'ReadFromOracle': 'beam:schematransform:org.apache.beam:jdbc_read:v1'
-        'WriteToOracle': 'beam:schematransform:org.apache.beam:jdbc_write:v1'
-        'ReadFromSqlServer': 'beam:schematransform:org.apache.beam:jdbc_read:v1'
-        'WriteToSqlServer': 'beam:schematransform:org.apache.beam:jdbc_write:v1'
       config:
         gradle_target: 'sdks:java:extensions:schemaio-expansion-service:shadowJar'
diff --git a/sdks/python/apache_beam/yaml/standard_providers.yaml b/sdks/python/apache_beam/yaml/standard_providers.yaml
index c612d4412081..67ab8da502f8 100644
--- a/sdks/python/apache_beam/yaml/standard_providers.yaml
+++ b/sdks/python/apache_beam/yaml/standard_providers.yaml
@@ -24,8 +24,6 @@
     version: BEAM_VERSION
   transforms:
      Sql: 'beam:external:java:sql:v1'
-     MapToFields-java: "beam:schematransform:org.apache.beam:yaml:map_to_fields-java:v1"
-     MapToFields-generic: "beam:schematransform:org.apache.beam:yaml:map_to_fields-java:v1"
 
 - type: renaming
   transforms:
@@ -35,6 +33,9 @@
     'Explode': 'Explode'
   config:
     mappings:
+      'Sql':
+        query: 'query'
+        # Unfortunately dialect is a java logical type.
       'MapToFields-generic':
         language: 'language'
         append: 'append'
@@ -57,6 +58,7 @@
     underlying_provider:
       type: beamJar
       transforms:
+        Sql: "schematransform:org.apache.beam:sql_transform:v1"
         MapToFields-java: "beam:schematransform:org.apache.beam:yaml:map_to_fields-java:v1"
         Filter-java: "beam:schematransform:org.apache.beam:yaml:filter-java:v1"
         Explode: "beam:schematransform:org.apache.beam:yaml:explode:v1"
diff --git a/sdks/python/apache_beam/yaml/yaml_io.py b/sdks/python/apache_beam/yaml/yaml_io.py
index bf4009719b80..1cd7231e35dd 100644
--- a/sdks/python/apache_beam/yaml/yaml_io.py
+++ b/sdks/python/apache_beam/yaml/yaml_io.py
@@ -52,11 +52,29 @@
 def read_from_text(path: str):
   # TODO(yaml): Consider passing the filename and offset, possibly even
   # by default.
+
+  """Reads lines from a text files.
+
+  The resulting PCollection consists of rows with a single string filed named
+  "line."
+
+  Args:
+    path (str): The file path to read from.  The path can contain glob
+      characters such as ``*`` and ``?``.
+  """
   return beam_io.ReadFromText(path) | beam.Map(lambda s: beam.Row(line=s))
 
 
 @beam.ptransform_fn
 def write_to_text(pcoll, path: str):
+  """Writes a PCollection to a (set of) text files(s).
+
+  The input must be a PCollection whose schema has exactly one field.
+
+  Args:
+      path (str): The file path to write to. The files written will
+        begin with this prefix, followed by a shard identifier.
+  """
   try:
     field_names = [
         name for name,
@@ -76,6 +94,11 @@ def write_to_text(pcoll, path: str):
 
 def read_from_bigquery(
     query=None, table=None, row_restriction=None, fields=None):
+  """Reads data from BigQuery.
+
+  Exactly one of table or query must be set.
+  If query is set, neither row_restriction nor fields should be set.
+  """
   if query is None:
     assert table is not None
   else:
@@ -95,6 +118,7 @@ def write_to_bigquery(
     create_disposition=BigQueryDisposition.CREATE_IF_NEEDED,
     write_disposition=BigQueryDisposition.WRITE_APPEND,
     error_handling=None):
+  """Writes data to a BigQuery table."""
   class WriteToBigQueryHandlingErrors(beam.PTransform):
     def default_label(self):
       return 'WriteToBigQuery'
@@ -152,10 +176,11 @@ def _create_parser(
   elif format == 'avro':
     beam_schema = avroio.avro_schema_to_beam_schema(schema)
     covert_to_row = avroio.avro_dict_to_beam_row(schema, beam_schema)
+    # pylint: disable=line-too-long
     return (
         beam_schema,
         lambda record: covert_to_row(
-            fastavro.schemaless_reader(io.BytesIO(record), schema)))
+            fastavro.schemaless_reader(io.BytesIO(record), schema)))  # type: ignore[call-arg]
   else:
     raise ValueError(f'Unknown format: {format}')
 
@@ -303,7 +328,7 @@ def write_to_pubsub(
     attributes_map: Optional[str] = None,
     id_attribute: Optional[str] = None,
     timestamp_attribute: Optional[str] = None):
-  """Writes messages from Cloud Pub/Sub.
+  """Writes messages to Cloud Pub/Sub.
 
   Args:
     topic: Cloud Pub/Sub topic in the form "/topics/<project>/<topic>".
diff --git a/sdks/python/apache_beam/yaml/yaml_mapping.py b/sdks/python/apache_beam/yaml/yaml_mapping.py
index 42af11ae2456..13b4d900791a 100644
--- a/sdks/python/apache_beam/yaml/yaml_mapping.py
+++ b/sdks/python/apache_beam/yaml/yaml_mapping.py
@@ -16,6 +16,8 @@
 #
 
 """This module defines the basic MapToFields operation."""
+import functools
+import inspect
 import itertools
 from collections import abc
 from typing import Any
@@ -23,6 +25,7 @@
 from typing import Collection
 from typing import Dict
 from typing import Mapping
+from typing import NamedTuple
 from typing import Optional
 from typing import Union
 
@@ -45,6 +48,17 @@
 from apache_beam.yaml.yaml_provider import dicts_to_rows
 
 
+def normalize_mapping(spec):
+  """
+  Normalizes various fields for mapping transforms.
+  """
+  if spec['type'] == 'MapToFields':
+    config = spec.get('config')
+    if isinstance(config.get('drop'), str):
+      config['drop'] = [config['drop']]
+  return spec
+
+
 def _check_mapping_arguments(
     transform_name, expression=None, callable=None, name=None, path=None):
   # Argument checking
@@ -267,6 +281,11 @@ def checking_func(row):
     return func
 
 
+class ErrorHandlingConfig(NamedTuple):
+  output: str
+  # TODO: Other parameters are valid here too, but not common to Java.
+
+
 def exception_handling_args(error_handling_spec):
   if error_handling_spec:
     return {
@@ -294,16 +313,52 @@ def expand(self, pcoll):
 
 
 def maybe_with_exception_handling_transform_fn(transform_fn):
+  @functools.wraps(transform_fn)
   def expand(pcoll, error_handling=None, **kwargs):
     wrapped_pcoll = beam.core._MaybePValueWithErrors(
         pcoll, exception_handling_args(error_handling))
     return transform_fn(wrapped_pcoll,
                         **kwargs).as_result(_map_errors_to_standard_format())
 
+  original_signature = inspect.signature(transform_fn)
+  new_parameters = list(original_signature.parameters.values())
+  error_handling_param = inspect.Parameter(
+      'error_handling',
+      inspect.Parameter.KEYWORD_ONLY,
+      default=None,
+      annotation=ErrorHandlingConfig)
+  if new_parameters[-1].kind == inspect.Parameter.VAR_KEYWORD:
+    new_parameters.insert(-1, error_handling_param)
+  else:
+    new_parameters.append(error_handling_param)
+  expand.__signature__ = original_signature.replace(parameters=new_parameters)
+
   return expand
 
 
 class _Explode(beam.PTransform):
+  """Explodes (aka unnest/flatten) one or more fields producing multiple rows.
+
+  Given one or more fields of iterable type, produces multiple rows, one for
+  each value of that field. For example, a row of the form `('a', [1, 2, 3])`
+  would expand to `('a', 1)`, `('a', 2')`, and `('a', 3)` when exploded on
+  the second field.
+
+  This is akin to a `FlatMap` when paired with the MapToFields transform.
+
+  Args:
+      fields: The list of fields to expand.
+      cross_product: If multiple fields are specified, indicates whether the
+          full cross-product of combinations should be produced, or if the
+          first element of the first field corresponds to the first element
+          of the second field, etc. For example, the row
+          `(['a', 'b'], [1, 2])` would expand to the four rows
+          `('a', 1)`, `('a', 2)`, `('b', 1)`, and `('b', 2)` when
+          `cross_product` is set to `true` but only the two rows
+          `('a', 1)` and `('b', 2)` when it is set to `false`.
+          Only meaningful (and required) if multiple rows are specified.
+      error_handling: Whether and how to handle errors during iteration.
+  """
   def __init__(
       self,
       fields: Union[str, Collection[str]],
@@ -409,8 +464,6 @@ def normalize_fields(pcoll, fields, drop=(), append=False, language='generic'):
       raise ValueError("Can only use expressions on a schema'd input.") from exn
     input_schema = {}
 
-  if isinstance(drop, str):
-    drop = [drop]
   if drop and not append:
     raise ValueError("Can only drop fields if append is true.")
   for name in drop:
diff --git a/sdks/python/apache_beam/yaml/yaml_provider.py b/sdks/python/apache_beam/yaml/yaml_provider.py
old mode 100644
new mode 100755
index 01e39b770c9b..4418b99341bc
--- a/sdks/python/apache_beam/yaml/yaml_provider.py
+++ b/sdks/python/apache_beam/yaml/yaml_provider.py
@@ -25,6 +25,7 @@
 import json
 import logging
 import os
+import re
 import subprocess
 import sys
 import urllib.parse
@@ -35,6 +36,7 @@
 from typing import Mapping
 from typing import Optional
 
+import docstring_parser
 import yaml
 from yaml.loader import SafeLoader
 
@@ -46,11 +48,9 @@
 from apache_beam.transforms import external
 from apache_beam.transforms import window
 from apache_beam.transforms.fully_qualified_named_transform import FullyQualifiedNamedTransform
-from apache_beam.typehints import native_type_compatibility
 from apache_beam.typehints import schemas
 from apache_beam.typehints import trivial_inference
 from apache_beam.typehints.schemas import named_tuple_to_schema
-from apache_beam.typehints.schemas import typing_from_runner_api
 from apache_beam.typehints.schemas import typing_to_runner_api
 from apache_beam.utils import python_callable
 from apache_beam.utils import subprocess_server
@@ -73,6 +73,9 @@ def provided_transforms(self) -> Iterable[str]:
   def config_schema(self, type):
     return None
 
+  def description(self, type):
+    return None
+
   def requires_inputs(self, typ: str, args: Mapping[str, Any]) -> bool:
     """Returns whether this transform requires inputs.
 
@@ -109,9 +112,9 @@ def affinity(self, other: "Provider"):
     # E.g. we could look at the the expected environments themselves.
     # Possibly, we could provide multiple expansions and have the runner itself
     # choose the actual implementation based on fusion (and other) criteria.
-    return (
-        self.underlying_provider()._affinity(other) +
-        other.underlying_provider()._affinity(self))
+    a = self.underlying_provider()
+    b = other.underlying_provider()
+    return a._affinity(b) + b._affinity(a)
 
   def _affinity(self, other: "Provider"):
     if self is other or self == other:
@@ -167,6 +170,10 @@ def config_schema(self, type):
       return named_tuple_to_schema(
           self.schema_transforms()[self._urns[type]].configuration_schema)
 
+  def description(self, type):
+    if self._urns[type] in self.schema_transforms():
+      return self.schema_transforms()[self._urns[type]].description
+
   def requires_inputs(self, typ, args):
     if self._urns[typ] in self.schema_transforms():
       return bool(self.schema_transforms()[self._urns[typ]].inputs)
@@ -412,7 +419,8 @@ def config_schema(self, typ):
     if isinstance(factory, type) and issubclass(factory, beam.PTransform):
       # https://bugs.python.org/issue40897
       params = dict(inspect.signature(factory.__init__).parameters)
-      del params['self']
+      if 'self' in params:
+        del params['self']
     else:
       params = inspect.signature(factory).parameters
 
@@ -423,15 +431,39 @@ def type_of(p):
       else:
         return t
 
+    docs = {
+        param.arg_name: param.description
+        for param in self.get_docs(typ).params
+    }
+
     names_and_types = [
         (name, typing_to_runner_api(type_of(p))) for name, p in params.items()
     ]
     return schema_pb2.Schema(
         fields=[
-            schema_pb2.Field(name=name, type=type) for name,
-            type in names_and_types
+            schema_pb2.Field(name=name, type=type, description=docs.get(name))
+            for (name, type) in names_and_types
         ])
 
+  def description(self, typ):
+    def empty_if_none(s):
+      return s or ''
+
+    docs = self.get_docs(typ)
+    return (
+        empty_if_none(docs.short_description) +
+        ('\n\n' if docs.blank_after_short_description else '\n') +
+        empty_if_none(docs.long_description)).strip() or None
+
+  def get_docs(self, typ):
+    docstring = self._transform_factories[typ].__doc__ or ''
+    # These "extra" docstring parameters are not relevant for YAML and mess
+    # up the parsing.
+    docstring = re.sub(
+        r'Pandas Parameters\s+-----.*', '', docstring, flags=re.S)
+    return docstring_parser.parse(
+        docstring, docstring_parser.DocstringStyle.GOOGLE)
+
   def create_transform(self, type, args, yaml_create_transform):
     return self._transform_factories[type](**args)
 
@@ -515,7 +547,7 @@ def dicts_to_rows(o):
 
 
 def create_builtin_provider():
-  def create(elements: Iterable[Any], reshuffle: bool = True):
+  def create(elements: Iterable[Any], reshuffle: Optional[bool] = True):
     """Creates a collection containing a specified set of elements.
 
     YAML/JSON-style mappings will be interpreted as Beam rows. For example::
@@ -529,17 +561,48 @@ def create(elements: Iterable[Any], reshuffle: bool = True):
     Args:
         elements: The set of elements that should belong to the PCollection.
             YAML/JSON-style mappings will be interpreted as Beam rows.
-        reshuffle (optional): Whether to introduce a reshuffle if there is more
-            than one element in the collection. Defaults to True.
+        reshuffle (optional): Whether to introduce a reshuffle (to possibly
+            redistribute the work) if there is more than one element in the
+            collection. Defaults to True.
     """
-    return beam.Create([element_to_rows(e) for e in elements], reshuffle)
+    return beam.Create([element_to_rows(e) for e in elements],
+                       reshuffle=reshuffle is not False)
 
   # Or should this be posargs, args?
   # pylint: disable=dangerous-default-value
   def fully_qualified_named_transform(
       constructor: str,
-      args: Iterable[Any] = (),
-      kwargs: Mapping[str, Any] = {}):
+      args: Optional[Iterable[Any]] = (),
+      kwargs: Optional[Mapping[str, Any]] = {}):
+    """A Python PTransform identified by fully qualified name.
+
+    This allows one to import, construct, and apply any Beam Python transform.
+    This can be useful for using transforms that have not yet been exposed
+    via a YAML interface. Note, however, that conversion may be required if this
+    transform does not accept or produce Beam Rows.
+
+    For example,
+
+        type: PyTransform
+        config:
+          constructor: apache_beam.pkg.mod.SomeClass
+          args: [1, 'foo']
+          kwargs:
+             baz: 3
+
+    can be used to access the transform
+    `apache_beam.pkg.mod.SomeClass(1, 'foo', baz=3)`.
+
+    Args:
+        constructor: Fully qualified name of a callable used to construct the
+            transform.  Often this is a class such as
+            `apache_beam.pkg.mod.SomeClass` but it can also be a function or
+            any other callable that returns a PTransform.
+        args: A list of parameters to pass to the callable as positional
+            arguments.
+        kwargs: A list of parameters to pass to the callable as keyword
+            arguments.
+    """
     with FullyQualifiedNamedTransform.with_filter('*'):
       return constructor >> FullyQualifiedNamedTransform(
           constructor, args, kwargs)
@@ -548,6 +611,19 @@ def fully_qualified_named_transform(
   # exactly zero or one PCollection in yaml (as they would be interpreted as
   # PBegin and the PCollection itself respectively).
   class Flatten(beam.PTransform):
+    """Flattens multiple PCollections into a single PCollection.
+
+    The elements of the resulting PCollection will be the (disjoint) union of
+    all the elements of all the inputs.
+
+    Note that in YAML transforms can always take a list of inputs which will
+    be implicitly flattened.
+    """
+    def __init__(self):
+      # Suppress the "label" argument from the superclass for better docs.
+      # pylint: disable=useless-parent-delegation
+      super().__init__()
+
     def expand(self, pcolls):
       if isinstance(pcolls, beam.PCollection):
         pipeline_arg = {}
@@ -561,6 +637,24 @@ def expand(self, pcolls):
       return pcolls | beam.Flatten(**pipeline_arg)
 
   class WindowInto(beam.PTransform):
+    # pylint: disable=line-too-long
+
+    """A window transform assigning windows to each element of a PCollection.
+
+    The assigned windows will affect all downstream aggregating operations,
+    which will aggregate by window as well as by key.
+
+    See [the Beam documentation on windowing](https://beam.apache.org/documentation/programming-guide/#windowing)
+    for more details.
+
+    Note that any Yaml transform can have a
+    [windowing parameter](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/yaml/README.md#windowing),
+    which is applied to its inputs (if any) or outputs (if there are no inputs)
+    which means that explicit WindowInto operations are not typically needed.
+
+    Args:
+      windowing: the type and parameters of the windowing to perform
+    """
     def __init__(self, windowing):
       self._window_transform = self._parse_window_spec(windowing)
 
@@ -586,13 +680,21 @@ def _parse_window_spec(spec):
       # TODO: Triggering, etc.
       return beam.WindowInto(window_fn)
 
-  def log_and_return(x):
-    logging.info(x)
-    return x
+  def LogForTesting():
+    """Logs each element of its input PCollection.
+
+    The output of this transform is a copy of its input for ease of use in
+    chain-style pipelines.
+    """
+    def log_and_return(x):
+      logging.info(x)
+      return x
+
+    return beam.Map(log_and_return)
 
   return InlineProvider({
       'Create': create,
-      'LogForTesting': lambda: beam.Map(log_and_return),
+      'LogForTesting': LogForTesting,
       'PyTransform': fully_qualified_named_transform,
       'Flatten': Flatten,
       'WindowInto': WindowInto,
@@ -727,16 +829,35 @@ def config_schema(self, type):
         self._transforms[type])
     if underlying_schema is None:
       return None
-    underlying_schema_types = {f.name: f.type for f in underlying_schema.fields}
+    defaults = self._defaults.get(type, {})
+    underlying_schema_fields = {f.name: f for f in underlying_schema.fields}
+    missing = set(self._mappings[type].values()) - set(
+        underlying_schema_fields.keys())
+    if missing:
+      raise ValueError(
+          f"Mapping destinations {missing} for {type} are not in the "
+          f"underlying config schema {list(underlying_schema_fields.keys())}")
+
+    def with_name(
+        original: schema_pb2.Field, new_name: str) -> schema_pb2.Field:
+      result = schema_pb2.Field()
+      result.CopyFrom(original)
+      result.name = new_name
+      return result
+
     return schema_pb2.Schema(
         fields=[
-            schema_pb2.Field(name=src, type=underlying_schema_types[dest])
-            for src,
-            dest in self._mappings[type].items()
+            with_name(underlying_schema_fields[dest], src)
+            for (src, dest) in self._mappings[type].items()
+            if dest not in defaults
         ])
 
+  def description(self, typ):
+    return self._underlying_provider.description(self._transforms[typ])
+
   def requires_inputs(self, typ, args):
-    return self._underlying_provider.requires_inputs(typ, args)
+    return self._underlying_provider.requires_inputs(
+        self._transforms[typ], args)
 
   def create_transform(
       self,
@@ -810,36 +931,3 @@ def standard_providers():
       create_combine_providers(),
       io_providers(),
       parse_providers(standard_providers))
-
-
-def list_providers():
-  def pretty_type(field_type):
-    if field_type.WhichOneof('type_info') == 'row_type':
-      return pretty_schema(field_type.row_type.schema)
-    else:
-      t = typing_from_runner_api(field_type)
-      optional_base = native_type_compatibility.extract_optional_type(t)
-      if optional_base:
-        t = optional_base
-        suffix = '?'
-      else:
-        suffix = ''
-      s = str(t)
-      if s.startswith('<class '):
-        s = t.__name__
-      return s + suffix
-
-  def pretty_schema(s):
-    if s is None:
-      return '[no schema]'
-    return 'Row(%s)' % ', '.join(
-        f'{f.name}={pretty_type(f.type)}' for f in s.fields)
-
-  for t, providers in sorted(standard_providers().items()):
-    print(t)
-    for p in providers:
-      print('\t', type(p).__name__, pretty_schema(p.config_schema(t)))
-
-
-if __name__ == '__main__':
-  list_providers()
diff --git a/sdks/python/apache_beam/yaml/yaml_transform.py b/sdks/python/apache_beam/yaml/yaml_transform.py
index ff5547db034c..5d71583e1932 100644
--- a/sdks/python/apache_beam/yaml/yaml_transform.py
+++ b/sdks/python/apache_beam/yaml/yaml_transform.py
@@ -36,6 +36,7 @@
 from apache_beam.transforms.fully_qualified_named_transform import FullyQualifiedNamedTransform
 from apache_beam.yaml import yaml_provider
 from apache_beam.yaml.yaml_combine import normalize_combine
+from apache_beam.yaml.yaml_mapping import normalize_mapping
 
 __all__ = ["YamlTransform"]
 
@@ -904,6 +905,7 @@ def preprocess_langauges(spec):
 
   for phase in [
       ensure_transforms_have_types,
+      normalize_mapping,
       normalize_combine,
       preprocess_langauges,
       ensure_transforms_have_providers,
@@ -962,7 +964,9 @@ def expand(self, pcolls):
     python_provider = yaml_provider.InlineProvider({})
 
     options = pipeline.options.view_as(GoogleCloudOptions)
-    options.labels = ["yaml=true"]
+    if not options.labels:
+      options.labels = []
+    options.labels += ["yaml=true"]
 
     result = expand_transform(
         self._spec,
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 7795e77e3963..ab6f75fc653b 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -99,11 +99,12 @@ platform_identifiers_map.each { platform, idsuffix ->
           environment CIBW_ENVIRONMENT: "SETUPTOOLS_USE_DISTUTILS=stdlib"
           // note: sync cibuildwheel version with GitHub Action
           // .github/workflow/build_wheel.yml:build_wheels "Install cibuildwheel" step
+          // note(https://github.com/pypa/cibuildwheel/issues/1692): cibuildwheel appears to timeout occasionally.
           executable 'sh'
           args '-c', ". ${envdir}/bin/activate && " +
               "pip install cibuildwheel==2.9.0 && " +
               "cibuildwheel --print-build-identifiers --platform ${platform} --archs ${archs} && " +
-              "cibuildwheel --output-dir ${buildDir} --platform ${platform} --archs ${archs}"
+              "for i in {1..3}; do cibuildwheel --output-dir ${buildDir} --platform ${platform} --archs ${archs} && break; done"
         }
       }
     }
@@ -162,4 +163,4 @@ tasks.register("wordCount") {
       args '-c', ". ${envdir}/bin/activate && python -m apache_beam.examples.wordcount --runner DirectRunner --output /tmp/output.txt"
     }
   }
-}
\ No newline at end of file
+}
diff --git a/sdks/python/container/Dockerfile b/sdks/python/container/Dockerfile
index a49933ee6604..7bea6229668f 100644
--- a/sdks/python/container/Dockerfile
+++ b/sdks/python/container/Dockerfile
@@ -17,7 +17,7 @@
 ###############################################################################
 
 ARG py_version
-FROM python:"${py_version}"-bullseye as beam
+FROM python:"${py_version}"-bookworm as beam
 LABEL Author "Apache Beam <dev@beam.apache.org>"
 ARG TARGETOS
 ARG TARGETARCH
diff --git a/sdks/python/container/base_image_requirements_manual.txt b/sdks/python/container/base_image_requirements_manual.txt
index f2f3ea44b44c..dd18da1c63f5 100644
--- a/sdks/python/container/base_image_requirements_manual.txt
+++ b/sdks/python/container/base_image_requirements_manual.txt
@@ -32,8 +32,7 @@ cython<1
 # some versions of libraries that launch Beam pipelines, like tensorflow-transform.
 # Leaving 'future' in our containers for now prevent breaking tft users.
 future
-# TODO(https://github.com/GoogleCloudPlatform/cloud-profiler-python/issues/127): Unpin python version.
-google-cloud-profiler;python_version<"3.11"
+google-cloud-profiler
 guppy3
 mmh3  # Optimizes execution of some Beam codepaths. TODO: Make it Beam's dependency.
 nltk  # Commonly used for natural language processing.
diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go
index ded10a44204a..c59807c72a98 100644
--- a/sdks/python/container/boot.go
+++ b/sdks/python/container/boot.go
@@ -409,6 +409,9 @@ func installSetupPackages(ctx context.Context, logger *tools.Logger, files []str
 	if err := pipInstallPackage(ctx, logger, files, workDir, workflowFile, false, true, nil); err != nil {
 		return fmt.Errorf("failed to install workflow: %v", err)
 	}
+	if err := logRuntimeDependencies(ctx, bufLogger); err != nil {
+		logger.Warnf(ctx, "couldn't fetch the runtime python dependencies: %v", err)
+	}
 
 	return nil
 }
@@ -453,3 +456,27 @@ func processArtifactsInSetupOnlyMode() {
 		log.Fatalf("Failed to install required packages: %v", setupErr)
 	}
 }
+
+// logRuntimeDependencies logs the python dependencies
+// installed in the runtime environment.
+func logRuntimeDependencies(ctx context.Context, bufLogger *tools.BufferedLogger) error {
+	bufLogger.Printf(ctx, "Logging runtime dependencies:")
+	pythonVersion, err := expansionx.GetPythonVersion()
+	if err != nil {
+		return err
+	}
+	bufLogger.Printf(ctx, "Using Python version:")
+	args := []string{"--version"}
+	if err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...); err != nil {
+		bufLogger.FlushAtError(ctx)
+	} else {
+		bufLogger.FlushAtDebug(ctx)
+	}
+	args = []string{"-m", "pip", "freeze"}
+	if err := execx.ExecuteEnvWithIO(nil, os.Stdin, bufLogger, bufLogger, pythonVersion, args...); err != nil {
+		bufLogger.FlushAtError(ctx)
+	} else {
+		bufLogger.FlushAtDebug(ctx)
+	}
+	return nil
+}
diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt
index a9f94104374e..636d546867fb 100644
--- a/sdks/python/container/py310/base_image_requirements.txt
+++ b/sdks/python/container/py310/base_image_requirements.txt
@@ -25,123 +25,124 @@ attrs==23.1.0
 beautifulsoup4==4.12.2
 bs4==0.0.1
 build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
 cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.4
-Cython==0.29.36
+cryptography==41.0.7
+Cython==0.29.37
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
-docker==6.1.3
+docker==7.0.0
 docopt==0.6.2
-exceptiongroup==1.1.3
+docstring-parser==0.15
+exceptiongroup==1.2.0
 execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.2
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.4.0
 future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.15.0
+google-api-python-client==2.111.0
 google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.25.2
 google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
-google-cloud-bigquery-storage==2.22.0
-google-cloud-bigtable==2.21.0
-google-cloud-core==2.3.3
-google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
-google-cloud-language==2.11.1
+google-cloud-aiplatform==1.38.1
+google-cloud-bigquery==3.14.1
+google-cloud-bigquery-storage==2.24.0
+google-cloud-bigtable==2.22.0
+google-cloud-core==2.4.1
+google-cloud-datastore==2.19.0
+google-cloud-dlp==3.14.0
+google-cloud-language==2.12.0
 google-cloud-profiler==4.1.0
-google-cloud-pubsub==2.18.4
-google-cloud-pubsublite==1.8.3
-google-cloud-recommendations-ai==0.10.5
-google-cloud-resource-manager==1.10.4
+google-cloud-pubsub==2.19.0
+google-cloud-pubsublite==1.9.0
+google-cloud-recommendations-ai==0.10.6
+google-cloud-resource-manager==1.11.0
 google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
-google-cloud-videointelligence==2.11.4
-google-cloud-vision==3.4.5
+google-cloud-storage==2.14.0
+google-cloud-videointelligence==2.12.0
+google-cloud-vision==3.5.0
 google-crc32c==1.5.0
-google-resumable-media==2.6.0
-googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
+greenlet==3.0.2
+grpc-google-iam-v1==0.13.0
+grpcio==1.60.0
+grpcio-status==1.60.0
+guppy3==3.1.4.post1
 hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.92.1
+idna==3.6
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonpickle==3.0.2
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
 numpy==1.24.4
 oauth2client==4.1.3
-objsize==0.6.1
-orjson==3.9.9
-overrides==6.5.0
+objsize==0.7.0
+orjson==3.9.10
+overrides==7.4.0
 packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
 pluggy==1.3.0
-proto-plus==1.22.3
-protobuf==4.25.0
+proto-plus==1.23.0
+protobuf==4.25.1
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
 pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.32.0
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.15.2
 rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
 shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
 soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
 tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tomli==2.0.1
 tqdm==4.66.1
-typing_extensions==4.8.0
-tzlocal==5.1
+typing_extensions==4.9.0
+tzlocal==5.2
 uritemplate==4.1.1
-urllib3==2.0.7
-websocket-client==1.6.4
-wrapt==1.15.0
-zstandard==0.21.0
+urllib3==2.1.0
+wrapt==1.16.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt
index 865b856683a4..c594f7f84870 100644
--- a/sdks/python/container/py311/base_image_requirements.txt
+++ b/sdks/python/container/py311/base_image_requirements.txt
@@ -25,117 +25,121 @@ attrs==23.1.0
 beautifulsoup4==4.12.2
 bs4==0.0.1
 build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
 cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.4
-Cython==0.29.36
+cryptography==41.0.7
+Cython==0.29.37
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
-docker==6.1.3
+docker==7.0.0
 docopt==0.6.2
+docstring-parser==0.15
 execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.2
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.4.0
 future==0.18.3
-google-api-core==2.12.0
+google-api-core==2.15.0
+google-api-python-client==2.111.0
 google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.25.2
 google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
-google-cloud-bigquery-storage==2.22.0
-google-cloud-bigtable==2.21.0
-google-cloud-core==2.3.3
-google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
-google-cloud-language==2.11.1
-google-cloud-pubsub==2.18.4
-google-cloud-pubsublite==1.8.3
-google-cloud-recommendations-ai==0.10.5
-google-cloud-resource-manager==1.10.4
+google-cloud-aiplatform==1.38.1
+google-cloud-bigquery==3.14.1
+google-cloud-bigquery-storage==2.24.0
+google-cloud-bigtable==2.22.0
+google-cloud-core==2.4.1
+google-cloud-datastore==2.19.0
+google-cloud-dlp==3.14.0
+google-cloud-language==2.12.0
+google-cloud-profiler==4.1.0
+google-cloud-pubsub==2.19.0
+google-cloud-pubsublite==1.9.0
+google-cloud-recommendations-ai==0.10.6
+google-cloud-resource-manager==1.11.0
 google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
-google-cloud-videointelligence==2.11.4
-google-cloud-vision==3.4.5
+google-cloud-storage==2.14.0
+google-cloud-videointelligence==2.12.0
+google-cloud-vision==3.5.0
 google-crc32c==1.5.0
-google-resumable-media==2.6.0
-googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
+greenlet==3.0.2
+grpc-google-iam-v1==0.13.0
+grpcio==1.60.0
+grpcio-status==1.60.0
+guppy3==3.1.4.post1
 hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
+hypothesis==6.92.1
+idna==3.6
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonpickle==3.0.2
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
 numpy==1.24.4
 oauth2client==4.1.3
-objsize==0.6.1
-orjson==3.9.9
-overrides==6.5.0
+objsize==0.7.0
+orjson==3.9.10
+overrides==7.4.0
 packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
 pluggy==1.3.0
-proto-plus==1.22.3
-protobuf==4.25.0
+proto-plus==1.23.0
+protobuf==4.25.1
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
 pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.32.0
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.15.2
 rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
 shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
 soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
 tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tqdm==4.66.1
-typing_extensions==4.8.0
-tzlocal==5.1
-urllib3==2.0.7
-websocket-client==1.6.4
-wrapt==1.15.0
-zstandard==0.21.0
+typing_extensions==4.9.0
+tzlocal==5.2
+uritemplate==4.1.1
+urllib3==2.1.0
+wrapt==1.16.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt
index 5dffff5f80d9..a0c27c0c3026 100644
--- a/sdks/python/container/py38/base_image_requirements.txt
+++ b/sdks/python/container/py38/base_image_requirements.txt
@@ -26,127 +26,128 @@ backports.zoneinfo==0.2.1
 beautifulsoup4==4.12.2
 bs4==0.0.1
 build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
 cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.4
-Cython==0.29.36
+cryptography==41.0.7
+Cython==0.29.37
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
-docker==6.1.3
+docker==7.0.0
 docopt==0.6.2
-exceptiongroup==1.1.3
+docstring-parser==0.15
+exceptiongroup==1.2.0
 execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.2
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.4.0
 future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.15.0
+google-api-python-client==2.111.0
 google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.25.2
 google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
-google-cloud-bigquery-storage==2.22.0
-google-cloud-bigtable==2.21.0
-google-cloud-core==2.3.3
-google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
-google-cloud-language==2.11.1
+google-cloud-aiplatform==1.38.1
+google-cloud-bigquery==3.14.1
+google-cloud-bigquery-storage==2.24.0
+google-cloud-bigtable==2.22.0
+google-cloud-core==2.4.1
+google-cloud-datastore==2.19.0
+google-cloud-dlp==3.14.0
+google-cloud-language==2.12.0
 google-cloud-profiler==4.1.0
-google-cloud-pubsub==2.18.4
-google-cloud-pubsublite==1.8.3
-google-cloud-recommendations-ai==0.10.5
-google-cloud-resource-manager==1.10.4
+google-cloud-pubsub==2.19.0
+google-cloud-pubsublite==1.9.0
+google-cloud-recommendations-ai==0.10.6
+google-cloud-resource-manager==1.11.0
 google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
-google-cloud-videointelligence==2.11.4
-google-cloud-vision==3.4.5
+google-cloud-storage==2.14.0
+google-cloud-videointelligence==2.12.0
+google-cloud-vision==3.5.0
 google-crc32c==1.5.0
-google-resumable-media==2.6.0
-googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
+greenlet==3.0.2
+grpc-google-iam-v1==0.13.0
+grpcio==1.60.0
+grpcio-status==1.60.0
+guppy3==3.1.4.post1
 hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
-importlib-metadata==6.8.0
-importlib-resources==6.1.0
+hypothesis==6.92.1
+idna==3.6
+importlib-metadata==7.0.0
+importlib-resources==6.1.1
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonpickle==3.0.2
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
 numpy==1.24.4
 oauth2client==4.1.3
-objsize==0.6.1
-orjson==3.9.9
-overrides==6.5.0
+objsize==0.7.0
+orjson==3.9.10
+overrides==7.4.0
 packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
 pkgutil_resolve_name==1.3.10
 pluggy==1.3.0
-proto-plus==1.22.3
-protobuf==4.25.0
+proto-plus==1.23.0
+protobuf==4.25.1
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
 pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.32.0
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.15.2
 rsa==4.9
-scikit-learn==1.3.1
+scikit-learn==1.3.2
 scipy==1.10.1
 shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
 soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
 tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tomli==2.0.1
 tqdm==4.66.1
-typing_extensions==4.8.0
-tzlocal==5.1
+typing_extensions==4.9.0
+tzlocal==5.2
 uritemplate==4.1.1
-urllib3==2.0.7
-websocket-client==1.6.4
-wrapt==1.15.0
+urllib3==2.1.0
+wrapt==1.16.0
 zipp==3.17.0
-zstandard==0.21.0
+zstandard==0.22.0
diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt
index 1b8ad7a2e748..953b7429a1a8 100644
--- a/sdks/python/container/py39/base_image_requirements.txt
+++ b/sdks/python/container/py39/base_image_requirements.txt
@@ -25,125 +25,126 @@ attrs==23.1.0
 beautifulsoup4==4.12.2
 bs4==0.0.1
 build==1.0.3
-cachetools==5.3.1
-certifi==2023.7.22
+cachetools==5.3.2
+certifi==2023.11.17
 cffi==1.16.0
-charset-normalizer==3.3.0
+charset-normalizer==3.3.2
 click==8.1.7
 cloudpickle==2.2.1
 crcmod==1.7
-cryptography==41.0.4
-Cython==0.29.36
+cryptography==41.0.7
+Cython==0.29.37
 deprecation==2.1.0
 dill==0.3.1.1
 dnspython==2.4.2
-docker==6.1.3
+docker==7.0.0
 docopt==0.6.2
-exceptiongroup==1.1.3
+docstring-parser==0.15
+exceptiongroup==1.2.0
 execnet==2.0.2
-fastavro==1.8.4
+fastavro==1.9.2
 fasteners==0.19
-freezegun==1.2.2
+freezegun==1.4.0
 future==0.18.3
-google-api-core==2.12.0
-google-api-python-client==2.104.0
+google-api-core==2.15.0
+google-api-python-client==2.111.0
 google-apitools==0.5.31
-google-auth==2.23.3
+google-auth==2.25.2
 google-auth-httplib2==0.1.1
-google-cloud-aiplatform==1.35.0
-google-cloud-bigquery==3.12.0
-google-cloud-bigquery-storage==2.22.0
-google-cloud-bigtable==2.21.0
-google-cloud-core==2.3.3
-google-cloud-datastore==2.18.0
-google-cloud-dlp==3.12.3
-google-cloud-language==2.11.1
+google-cloud-aiplatform==1.38.1
+google-cloud-bigquery==3.14.1
+google-cloud-bigquery-storage==2.24.0
+google-cloud-bigtable==2.22.0
+google-cloud-core==2.4.1
+google-cloud-datastore==2.19.0
+google-cloud-dlp==3.14.0
+google-cloud-language==2.12.0
 google-cloud-profiler==4.1.0
-google-cloud-pubsub==2.18.4
-google-cloud-pubsublite==1.8.3
-google-cloud-recommendations-ai==0.10.5
-google-cloud-resource-manager==1.10.4
+google-cloud-pubsub==2.19.0
+google-cloud-pubsublite==1.9.0
+google-cloud-recommendations-ai==0.10.6
+google-cloud-resource-manager==1.11.0
 google-cloud-spanner==3.40.1
-google-cloud-storage==2.12.0
-google-cloud-videointelligence==2.11.4
-google-cloud-vision==3.4.5
+google-cloud-storage==2.14.0
+google-cloud-videointelligence==2.12.0
+google-cloud-vision==3.5.0
 google-crc32c==1.5.0
-google-resumable-media==2.6.0
-googleapis-common-protos==1.61.0
-greenlet==3.0.0
-grpc-google-iam-v1==0.12.6
-grpcio==1.59.0
-grpcio-status==1.59.0
-guppy3==3.1.4
+google-resumable-media==2.7.0
+googleapis-common-protos==1.62.0
+greenlet==3.0.2
+grpc-google-iam-v1==0.13.0
+grpcio==1.60.0
+grpcio-status==1.60.0
+guppy3==3.1.4.post1
 hdfs==2.7.3
 httplib2==0.22.0
-hypothesis==6.88.1
-idna==3.4
-importlib-metadata==6.8.0
+hypothesis==6.92.1
+idna==3.6
+importlib-metadata==7.0.0
 iniconfig==2.0.0
 joblib==1.3.2
 Js2Py==0.74
-jsonschema==4.19.1
-jsonschema-specifications==2023.7.1
+jsonpickle==3.0.2
+jsonschema==4.20.0
+jsonschema-specifications==2023.11.2
 mmh3==4.0.1
 mock==5.1.0
 nltk==3.8.1
 nose==1.3.7
 numpy==1.24.4
 oauth2client==4.1.3
-objsize==0.6.1
-orjson==3.9.9
-overrides==6.5.0
+objsize==0.7.0
+orjson==3.9.10
+overrides==7.4.0
 packaging==23.2
 pandas==1.5.3
 parameterized==0.9.0
 pluggy==1.3.0
-proto-plus==1.22.3
-protobuf==4.25.0
+proto-plus==1.23.0
+protobuf==4.25.1
 psycopg2-binary==2.9.9
-pyarrow==11.0.0
-pyarrow-hotfix==0.4
-pyasn1==0.5.0
+pyarrow==14.0.2
+pyarrow-hotfix==0.6
+pyasn1==0.5.1
 pyasn1-modules==0.3.0
 pycparser==2.21
 pydot==1.4.2
-PyHamcrest==2.0.4
+PyHamcrest==2.1.0
 pyjsparser==2.7.1
-pymongo==4.5.0
+pymongo==4.6.1
 PyMySQL==1.1.0
 pyparsing==3.1.1
 pyproject_hooks==1.0.0
-pytest==7.4.2
+pytest==7.4.3
 pytest-timeout==2.2.0
-pytest-xdist==3.3.1
+pytest-xdist==3.5.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2023.3.post1
 PyYAML==6.0.1
-referencing==0.30.2
+referencing==0.32.0
 regex==2023.10.3
 requests==2.31.0
 requests-mock==1.11.0
-rpds-py==0.10.6
+rpds-py==0.15.2
 rsa==4.9
-scikit-learn==1.3.1
-scipy==1.11.3
+scikit-learn==1.3.2
+scipy==1.11.4
 shapely==2.0.2
 six==1.16.0
 sortedcontainers==2.4.0
 soupsieve==2.5
-SQLAlchemy==1.4.49
+SQLAlchemy==1.4.50
 sqlparse==0.4.4
 tenacity==8.2.3
 testcontainers==3.7.1
 threadpoolctl==3.2.0
 tomli==2.0.1
 tqdm==4.66.1
-typing_extensions==4.8.0
-tzlocal==5.1
+typing_extensions==4.9.0
+tzlocal==5.2
 uritemplate==4.1.1
-urllib3==2.0.7
-websocket-client==1.6.4
-wrapt==1.15.0
+urllib3==2.1.0
+wrapt==1.16.0
 zipp==3.17.0
-zstandard==0.21.0
+zstandard==0.22.0
diff --git a/sdks/python/expansion-service-container/Dockerfile b/sdks/python/expansion-service-container/Dockerfile
index c6a33cdae88e..d3cd4a4afad3 100644
--- a/sdks/python/expansion-service-container/Dockerfile
+++ b/sdks/python/expansion-service-container/Dockerfile
@@ -18,7 +18,7 @@
 
 # We just need to support one Python version supported by Beam.
 # Picking the current default Beam Python version which is Python 3.8.
-FROM python:3.8-bullseye as expansion-service
+FROM python:3.8-bookworm as expansion-service
 LABEL Author "Apache Beam <dev@beam.apache.org>"
 ARG TARGETOS
 ARG TARGETARCH
diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py
index 2b488af0afb5..a2cd1bd4cef3 100644
--- a/sdks/python/gen_protos.py
+++ b/sdks/python/gen_protos.py
@@ -527,7 +527,6 @@ def generate_proto_files(force=False):
   generate_init_files_lite(PYTHON_OUTPUT_PATH)
   for proto_package in proto_packages:
     generate_urn_files(proto_package, PYTHON_OUTPUT_PATH)
-
     generate_init_files_full(PYTHON_OUTPUT_PATH)
 
 
diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini
index 140476b29e50..4ffbb4524c06 100644
--- a/sdks/python/pytest.ini
+++ b/sdks/python/pytest.ini
@@ -33,7 +33,7 @@ markers =
     uses_gcp_java_expansion_service: collect Cross Language GCP Java transforms test runs
     uses_java_expansion_service: collect Cross Language Java transforms test runs
     uses_python_expansion_service: collect Cross Language Python transforms test runs
-    uses_io_expansion_service: collect Cross Language transform test runs (with Kafka bootstrap server)
+    uses_io_java_expansion_service: collect Cross Language IO Java transform test runs (with Kafka bootstrap server)
     uses_transform_service: collect Cross Language test runs that uses the Transform Service
     xlang_sql_expansion_service: collect for Cross Language with SQL expansion service test runs
     it_postcommit: collect for post-commit integration test runs
diff --git a/sdks/python/scripts/generate_pydoc.sh b/sdks/python/scripts/generate_pydoc.sh
index 06ad06320fcf..82740ae67c9f 100755
--- a/sdks/python/scripts/generate_pydoc.sh
+++ b/sdks/python/scripts/generate_pydoc.sh
@@ -133,7 +133,9 @@ autodoc_inherit_docstrings = False
 autodoc_member_order = 'bysource'
 autodoc_mock_imports = ["tensorrt", "cuda", "torch",
     "onnxruntime", "onnx", "tensorflow", "tensorflow_hub",
-    "tensorflow_transform", "tensorflow_metadata", "transformers"]
+    "tensorflow_transform", "tensorflow_metadata", "transformers", "xgboost", "datatable", "transformers",
+    "sentence_transformers",
+    ]
 
 # Allow a special section for documenting DataFrame API
 napoleon_custom_sections = ['Differences from pandas']
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 1785cd75df80..e8c93ce85484 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -141,9 +141,21 @@ def cythonize(*args, **kwargs):
 # [BEAM-8181] pyarrow cannot be installed on 32-bit Windows platforms.
 if sys.platform == 'win32' and sys.maxsize <= 2**32:
   pyarrow_dependency = ['']
+elif sys.platform == 'win32' or sys.platform == 'cygwin':
+  # https://github.com/apache/beam/issues/28410 - pyarrow>=13 seeing issues
+  # on windows with error
+  # C:\arrow\cpp\src\arrow\filesystem\s3fs.cc:2904:  arrow::fs::FinalizeS3 was
+  # not called even though S3 was initialized.  This could lead to a
+  # segmentation fault at exit. Keep pyarrow<13 until this is resolved.
+  pyarrow_dependency = [
+        'pyarrow>=3.0.0,<12.0.0',
+        # NOTE: We can remove this once Beam increases the pyarrow lower bound
+        # to a version that fixes CVE.
+        'pyarrow-hotfix<1'
+    ]
 else:
   pyarrow_dependency = [
-      'pyarrow>=3.0.0,<12.0.0',
+      'pyarrow>=3.0.0,<15.0.0',
       # NOTE(https://github.com/apache/beam/issues/29392): We can remove this
       # once Beam increases the pyarrow lower bound to a version that fixes CVE.
       'pyarrow-hotfix<1'
@@ -286,10 +298,11 @@ def get_portability_package_data():
           'httplib2>=0.8,<0.23.0',
           'js2py>=0.74,<1',
           'jsonschema>=4.0.0,<5.0.0',
+          'jsonpickle>=3.0.0,<4.0.0',
           # numpy can have breaking changes in minor versions.
           # Use a strict upper bound.
           'numpy>=1.14.3,<1.25.0',  # Update pyproject.toml as well.
-          'objsize>=0.6.1,<0.7.0',
+          'objsize>=0.6.1,<0.8.0',
           'packaging>=22.0',
           'pymongo>=3.8.0,<5.0.0',
           'proto-plus>=1.7.1,<2',
@@ -320,12 +333,14 @@ def get_portability_package_data():
       extras_require={
           'docs': [
               'Sphinx>=1.5.2,<2.0',
+              'docstring-parser>=0.15,<1.0',
               # Pinning docutils as a workaround for Sphinx issue:
               # https://github.com/sphinx-doc/sphinx/issues/9727
               'docutils==0.17.1',
               'pandas<2.0.0',
           ],
           'test': [
+              'docstring-parser>=0.15,<1.0',
               'freezegun>=0.3.12',
               'joblib>=1.0.1',
               'mock>=1.0.1,<6.0.0',
@@ -344,6 +359,7 @@ def get_portability_package_data():
               'testcontainers[mysql]>=3.0.3,<4.0.0',
               'cryptography>=41.0.2',
               'hypothesis>5.0.0,<=7.0.0',
+              'pyyaml>=3.12,<7.0.0',
           ],
           'gcp': [
               'cachetools>=3.1.0,<6',
@@ -357,7 +373,7 @@ def get_portability_package_data():
               'google-cloud-datastore>=2.0.0,<3',
               'google-cloud-pubsub>=2.1.0,<3',
               'google-cloud-pubsublite>=1.2.0,<2',
-              'google-cloud-storage>=2.10.0,<3',
+              'google-cloud-storage>=2.14.0,<3',
               # GCP packages required by tests
               'google-cloud-bigquery>=2.0.0,<4',
               'google-cloud-bigquery-storage>=2.6.3,<3',
@@ -406,6 +422,7 @@ def get_portability_package_data():
               'distributed >= 2022.6',
           ],
           'yaml': [
+              'docstring-parser>=0.15,<1.0',
               'pyyaml>=3.12,<7.0.0',
           ] + dataframe_dependency
       },
diff --git a/sdks/python/test-suites/dataflow/build.gradle b/sdks/python/test-suites/dataflow/build.gradle
index b55716a42df2..04a79683fd36 100644
--- a/sdks/python/test-suites/dataflow/build.gradle
+++ b/sdks/python/test-suites/dataflow/build.gradle
@@ -67,13 +67,13 @@ task examplesPostCommit {
 }
 
 task gcpCrossLanguagePostCommit {
-  getVersionsAsList('cross_language_validates_gcp_py_versions').each {
+  getVersionsAsList('cross_language_validates_py_versions').each {
     dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:gcpCrossLanguagePythonUsingJava")
   }
 }
 
 task ioCrossLanguagePostCommit {
-  getVersionsAsList('cross_language_validates_gcp_py_versions').each {
+  getVersionsAsList('cross_language_validates_py_versions').each {
     dependsOn.add(":sdks:python:test-suites:dataflow:py${getVersionSuffix(it)}:ioCrossLanguagePythonUsingJava")
   }
 }
diff --git a/sdks/python/test-suites/direct/build.gradle b/sdks/python/test-suites/direct/build.gradle
index fc408411ec22..ea643c3303aa 100644
--- a/sdks/python/test-suites/direct/build.gradle
+++ b/sdks/python/test-suites/direct/build.gradle
@@ -32,7 +32,13 @@ tasks.register("examplesPostCommit") {
 }
 
 task gcpCrossLanguagePostCommit {
-  getVersionsAsList('cross_language_validates_gcp_py_versions').each {
+  getVersionsAsList('cross_language_validates_py_versions').each {
     dependsOn.add(":sdks:python:test-suites:direct:py${getVersionSuffix(it)}:gcpCrossLanguagePythonUsingJava")
   }
 }
+
+task ioCrossLanguagePostCommit {
+  getVersionsAsList('cross_language_validates_py_versions').each {
+    dependsOn.add(":sdks:python:test-suites:direct:py${getVersionSuffix(it)}:ioCrossLanguagePythonUsingJava")
+  }
+}
diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle
index da7aea95e1b0..771e0be19cfe 100644
--- a/sdks/python/test-suites/direct/common.gradle
+++ b/sdks/python/test-suites/direct/common.gradle
@@ -194,7 +194,7 @@ tasks.register("azureIntegrationTest") {
   doLast {
     exec {
       executable 'sh'
-      args '-c', "${rootDir}/sdks/python/apache_beam/io/azure/integration_test/azure_integration_test.sh python:${pythonContainerVersion}-bullseye"
+      args '-c', "${rootDir}/sdks/python/apache_beam/io/azure/integration_test/azure_integration_test.sh python:${pythonContainerVersion}-bookworm"
     }
   }
 }
diff --git a/sdks/python/test-suites/gradle.properties b/sdks/python/test-suites/gradle.properties
index 72fc651733db..c4e94bed4d84 100644
--- a/sdks/python/test-suites/gradle.properties
+++ b/sdks/python/test-suites/gradle.properties
@@ -47,5 +47,5 @@ samza_validates_runner_postcommit_py_versions=3.8,3.11
 # spark runner test-suites
 spark_examples_postcommit_py_versions=3.8,3.11
 
-# cross language gcp io postcommit python test suites
-cross_language_validates_gcp_py_versions=3.8,3.11
+# cross language postcommit python test suites
+cross_language_validates_py_versions=3.8,3.11
diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle
index b1ed5f88c7c9..1e03b5058083 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -141,6 +141,10 @@ toxTask "testPy38transformers-430", "py38-transformers-430", "${posargs}"
 test.dependsOn "testPy38transformers-430"
 preCommitPyCoverage.dependsOn "testPy38transformers-430"
 
+toxTask "testPy38embeddingsMLTransform", "py38-embeddings", "${posargs}"
+test.dependsOn "testPy38embeddingsMLTransform"
+preCommitPyCoverage.dependsOn "testPy38embeddingsMLTransform"
+
 toxTask "whitespacelint", "whitespacelint", "${posargs}"
 
 task archiveFilesToLint(type: Zip) {
diff --git a/sdks/python/test-suites/xlang/build.gradle b/sdks/python/test-suites/xlang/build.gradle
index df3ebdd1582c..5a124ac20ce2 100644
--- a/sdks/python/test-suites/xlang/build.gradle
+++ b/sdks/python/test-suites/xlang/build.gradle
@@ -56,20 +56,17 @@ def gcpXlangCommon = new CrossLanguageTaskCommon().tap {
     startJobServer = setupTask
     cleanupJobServer = cleanupTask
 }
-xlangTasks.add(gcpXlangCommon)
-
-def ioExpansionProject = project.project(':sdks:java:io:expansion-service')
 
 def ioXlangCommon = new CrossLanguageTaskCommon().tap {
     name = "ioCrossLanguage"
-    expansionProjectPath = ioExpansionProject.getPath()
-    collectMarker = "uses_io_expansion_service"
+    expansionProjectPath = project.project(':sdks:java:io:expansion-service').getPath()
+    collectMarker = "uses_io_java_expansion_service"
     startJobServer = setupTask
     cleanupJobServer = cleanupTask
     //See .test-infra/kafka/bitnami/README.md for setup instructions
     additionalEnvs = ["KAFKA_BOOTSTRAP_SERVER":project.findProperty('kafkaBootstrapServer')]
 }
 
-xlangTasks.add(ioXlangCommon)
+xlangTasks.addAll(gcpXlangCommon, ioXlangCommon)
 
 ext.xlangTasks = xlangTasks
\ No newline at end of file
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index e4cf09cacba4..dbe90c084af2 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -148,10 +148,6 @@ deps =
   sphinx_rtd_theme==0.4.3
   docutils<0.18
   Jinja2==3.0.3 # TODO(https://github.com/apache/beam/issues/21587): Sphinx version is too old.
-  torch
-  xgboost<=1.7.6
-  datatable==1.0.0
-  transformers
 commands =
   time {toxinidir}/scripts/generate_pydoc.sh
 
@@ -423,3 +419,15 @@ commands =
   # Run all Vertex AI unit tests
   # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_vertex_ai {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
+
+
+[testenv:py{38,39,310,311}-embeddings]
+deps =
+  sentence-transformers==2.2.2
+extras = test,gcp
+commands =
+  # Log aiplatform and its dependencies version for debugging
+  /bin/sh -c "pip freeze | grep -E sentence-transformers"
+  /bin/sh -c "pip freeze | grep -E google-cloud-aiplatform"
+  # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories.
+  /bin/sh -c 'pytest apache_beam/ml/transforms/embeddings -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
diff --git a/sdks/typescript/package-lock.json b/sdks/typescript/package-lock.json
index 55d510ccce8c..1eb77b748005 100644
--- a/sdks/typescript/package-lock.json
+++ b/sdks/typescript/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "apache-beam",
-  "version": "2.52.0-SNAPSHOT",
+  "version": "2.54.0-SNAPSHOT",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "apache-beam",
-      "version": "2.52.0-SNAPSHOT",
+      "version": "2.54.0-SNAPSHOT",
       "dependencies": {
         "@google-cloud/pubsub": "^2.19.4",
         "@grpc/grpc-js": "~1.4.6",
@@ -39,7 +39,7 @@
         "istanbul": "^0.4.5",
         "js-yaml": "^4.1.0",
         "mocha": "^9.1.3",
-        "prettier": "^2.5.1",
+        "prettier": "^3.1.1",
         "typedoc": "^0.23.23",
         "typescript": "4.7"
       }
@@ -3193,15 +3193,18 @@
       }
     },
     "node_modules/prettier": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz",
-      "integrity": "sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz",
+      "integrity": "sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==",
       "dev": true,
       "bin": {
-        "prettier": "bin-prettier.js"
+        "prettier": "bin/prettier.cjs"
       },
       "engines": {
-        "node": ">=10.13.0"
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/prettier/prettier?sponsor=1"
       }
     },
     "node_modules/proto3-json-serializer": {
@@ -6398,9 +6401,9 @@
       "dev": true
     },
     "prettier": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.5.1.tgz",
-      "integrity": "sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==",
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.1.tgz",
+      "integrity": "sha512-22UbSzg8luF4UuZtzgiUOfcGM8s4tjBv6dJRT7j275NXsy2jb4aJa4NNveul5x4eqlF1wuhuR2RElK71RvmVaw==",
       "dev": true
     },
     "proto3-json-serializer": {
diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json
index 918846a79add..8fc999bc9c9d 100644
--- a/sdks/typescript/package.json
+++ b/sdks/typescript/package.json
@@ -1,6 +1,6 @@
 {
   "name": "apache-beam",
-  "version": "2.53.0-SNAPSHOT",
+  "version": "2.54.0-SNAPSHOT",
   "devDependencies": {
     "@google-cloud/bigquery": "^5.12.0",
     "@types/mocha": "^9.0.0",
@@ -11,7 +11,7 @@
     "istanbul": "^0.4.5",
     "js-yaml": "^4.1.0",
     "mocha": "^9.1.3",
-    "prettier": "^2.5.1",
+    "prettier": "^3.1.1",
     "typedoc": "^0.23.23",
     "typescript": "4.7"
   },
diff --git a/sdks/typescript/src/apache_beam/coders/coders.ts b/sdks/typescript/src/apache_beam/coders/coders.ts
index 780987138fbb..1cf19b01ff2e 100644
--- a/sdks/typescript/src/apache_beam/coders/coders.ts
+++ b/sdks/typescript/src/apache_beam/coders/coders.ts
@@ -70,7 +70,7 @@ class CoderRegistry {
 
   registerConstructor(
     urn: string,
-    constructor: (...args: unknown[]) => Coder<unknown>
+    constructor: (...args: unknown[]) => Coder<unknown>,
   ) {
     this.internal_registry[urn] = constructor;
   }
@@ -142,7 +142,7 @@ export interface Coder<T> {
 function writeByteCallback(
   val: number,
   buf: { [x: string]: number },
-  pos: number
+  pos: number,
 ) {
   buf[pos] = val & 0xff;
 }
@@ -162,7 +162,7 @@ export function writeRawByte(b: unknown, writer: HackedWriter) {
 function writeBytesCallback(
   val: number[],
   buf: { [x: string]: number },
-  pos: number
+  pos: number,
 ) {
   for (let i = 0; i < val.length; ++i) {
     buf[pos + i] = val[i];
diff --git a/sdks/typescript/src/apache_beam/coders/required_coders.ts b/sdks/typescript/src/apache_beam/coders/required_coders.ts
index fe852b349251..74b7154ba114 100644
--- a/sdks/typescript/src/apache_beam/coders/required_coders.ts
+++ b/sdks/typescript/src/apache_beam/coders/required_coders.ts
@@ -321,7 +321,7 @@ export class IterableCoder<T> implements Coder<Iterable<T>> {
         }
         for (var i = 0; i < count; i++) {
           result.push(
-            this.elementCoder.decode(reader, Context.needsDelimiters)
+            this.elementCoder.decode(reader, Context.needsDelimiters),
           );
         }
       }
@@ -369,7 +369,7 @@ export class LengthPrefixedCoder<T> implements Coder<T> {
   decode(reader: Reader, context: Context): T {
     return this.elementCoder.decode(
       new Reader(reader.bytes()),
-      Context.wholeStream
+      Context.wholeStream,
     );
   }
 }
@@ -384,7 +384,10 @@ export class FullWindowedValueCoder<T, W extends Window>
   static URN: string = "beam:coder:windowed_value:v1";
   windowIterableCoder: IterableCoder<W>; // really W
 
-  constructor(public elementCoder: Coder<T>, public windowCoder: Coder<W>) {
+  constructor(
+    public elementCoder: Coder<T>,
+    public windowCoder: Coder<W>,
+  ) {
     this.windowIterableCoder = new IterableCoder(windowCoder);
   }
 
@@ -405,17 +408,17 @@ export class FullWindowedValueCoder<T, W extends Window>
     InstantCoder.INSTANCE.encode(
       windowedValue.timestamp,
       writer,
-      Context.needsDelimiters
+      Context.needsDelimiters,
     );
     this.windowIterableCoder.encode(
       <Array<W>>windowedValue.windows,
       writer,
-      Context.needsDelimiters
+      Context.needsDelimiters,
     ); // Windows.
     PaneInfoCoder.INSTANCE.encode(
       windowedValue.pane,
       writer,
-      Context.needsDelimiters
+      Context.needsDelimiters,
     );
     this.elementCoder.encode(windowedValue.value, writer, context);
   }
@@ -423,11 +426,11 @@ export class FullWindowedValueCoder<T, W extends Window>
   decode(reader: Reader, context: Context): WindowedValue<T> {
     const timestamp = InstantCoder.INSTANCE.decode(
       reader,
-      Context.needsDelimiters
+      Context.needsDelimiters,
     );
     const windows = this.windowIterableCoder.decode(
       reader,
-      Context.needsDelimiters
+      Context.needsDelimiters,
     );
     const pane = PaneInfoCoder.INSTANCE.decode(reader, Context.needsDelimiters);
     const value = this.elementCoder.decode(reader, context);
@@ -472,8 +475,8 @@ export class InstantCoder implements Coder<Instant> {
   decode(reader: Reader, context: Context): Instant {
     const shiftedMillis = Long.fromBytesBE(
       Array.from(
-        reader.buf.slice(reader.pos, reader.pos + InstantCoder.INSTANT_BYTES)
-      )
+        reader.buf.slice(reader.pos, reader.pos + InstantCoder.INSTANT_BYTES),
+      ),
     );
     reader.pos += InstantCoder.INSTANT_BYTES;
     return shiftedMillis.add(Long.MIN_VALUE);
@@ -503,7 +506,7 @@ export class PaneInfoCoder implements Coder<PaneInfo> {
   static INSTANCE = new PaneInfoCoder();
   static ONE_AND_ONLY_FIRING = PaneInfoCoder.INSTANCE.decode(
     new Reader(new Uint8Array([0x09])),
-    null!
+    null!,
   );
 
   private static decodeTiming(timingNumber): Timing {
@@ -520,7 +523,7 @@ export class PaneInfoCoder implements Coder<PaneInfo> {
         throw new Error(
           "Timing number 0b" +
             timingNumber.toString(2) +
-            " has more than two bits of info"
+            " has more than two bits of info",
         );
     }
   }
@@ -635,7 +638,7 @@ export class PaneInfoCoder implements Coder<PaneInfo> {
 
   toProto(pipelineContext: ProtoContext): runnerApi.Coder {
     throw new Error(
-      "No proto encoding for PaneInfoCoder, always part of WindowedValue codec"
+      "No proto encoding for PaneInfoCoder, always part of WindowedValue codec",
     );
   }
 }
diff --git a/sdks/typescript/src/apache_beam/coders/row_coder.ts b/sdks/typescript/src/apache_beam/coders/row_coder.ts
index f93abe1f5ece..6c2b1f1a75d9 100644
--- a/sdks/typescript/src/apache_beam/coders/row_coder.ts
+++ b/sdks/typescript/src/apache_beam/coders/row_coder.ts
@@ -129,8 +129,8 @@ export class RowCoder implements Coder<any> {
         default:
           throw new Error(
             `Encountered a type that is not currently supported by RowCoder: ${JSON.stringify(
-              f.type
-            )}`
+              f.type,
+            )}`,
           );
       }
       return obj;
@@ -265,14 +265,14 @@ export class RowCoder implements Coder<any> {
             return new BoolCoder();
           default:
             throw new Error(
-              `Encountered an Atomic type that is not currently supported by RowCoder: ${atomicType}`
+              `Encountered an Atomic type that is not currently supported by RowCoder: ${atomicType}`,
             );
         }
         break;
       case "arrayType":
         if (typeInfo.arrayType.elementType !== undefined) {
           return new IterableCoder(
-            this.getCoderFromType(typeInfo.arrayType.elementType)
+            this.getCoderFromType(typeInfo.arrayType.elementType),
           );
         } else {
           throw new Error("ElementType missing on ArrayType");
@@ -290,14 +290,14 @@ export class RowCoder implements Coder<any> {
         const logicalTypeInfo = logicalTypes.get(typeInfo.logicalType.urn);
         if (logicalTypeInfo !== undefined) {
           const reprCoder = this.getCoderFromType(
-            typeInfo.logicalType.representation!
+            typeInfo.logicalType.representation!,
           );
           return {
             encode: (element: any, writer: Writer, context: Context) =>
               reprCoder.encode(
                 logicalTypeInfo.toRepr(element),
                 writer,
-                context
+                context,
               ),
             decode: (reader: Reader, context: Context) =>
               logicalTypeInfo.fromRepr(reprCoder.decode(reader, context)),
@@ -312,8 +312,8 @@ export class RowCoder implements Coder<any> {
       default:
         throw new Error(
           `Encountered a type that is not currently supported by RowCoder: ${JSON.stringify(
-            t
-          )}`
+            t,
+          )}`,
         );
     }
   }
@@ -342,7 +342,7 @@ export class RowCoder implements Coder<any> {
       let encPosx = schema.fields.map((f: Field) => f.encodingPosition);
       if (encPosx.length !== this.encodingPositions.length) {
         throw new Error(
-          `Schema with id ${this.schema.id} has encoding_positions_set=True, but not all fields have encoding_position set`
+          `Schema with id ${this.schema.id} has encoding_positions_set=True, but not all fields have encoding_position set`,
         );
       }
       // Checking if positions are in {0, ..., length-1}
@@ -352,7 +352,7 @@ export class RowCoder implements Coder<any> {
     }
 
     this.hasNullableFields = this.schema.fields.some(
-      (f: Field) => f.type?.nullable
+      (f: Field) => f.type?.nullable,
     );
     this.components = this.encodingPositions
       .map((i) => this.schema.fields[i])
@@ -409,7 +409,7 @@ export class RowCoder implements Coder<any> {
       if (attr === null || attr === undefined) {
         if (!this.fieldNullable[i]) {
           throw new Error(
-            `Attempted to encode null for non-nullable field \"${this.schema.fields[i].name}\".`
+            `Attempted to encode null for non-nullable field \"${this.schema.fields[i].name}\".`,
           );
         }
       } else {
@@ -472,7 +472,7 @@ export class RowCoder implements Coder<any> {
       obj = this.addFieldOfType(
         obj,
         this.schema.fields[i],
-        sortedComponents[i]
+        sortedComponents[i],
       );
     });
 
diff --git a/sdks/typescript/src/apache_beam/coders/standard_coders.ts b/sdks/typescript/src/apache_beam/coders/standard_coders.ts
index a9714247f8e5..8f78bc67b828 100644
--- a/sdks/typescript/src/apache_beam/coders/standard_coders.ts
+++ b/sdks/typescript/src/apache_beam/coders/standard_coders.ts
@@ -225,5 +225,5 @@ globalRegistry().register(IntervalWindowCoder.URN, IntervalWindowCoder);
 import { requireForSerialization } from "../serialization";
 requireForSerialization(
   "apache-beam/coders/standard_coders",
-  exports as Record<string, unknown>
+  exports as Record<string, unknown>,
 );
diff --git a/sdks/typescript/src/apache_beam/examples/wordcount.ts b/sdks/typescript/src/apache_beam/examples/wordcount.ts
index e34b8cab3c5c..3ba11878b22d 100644
--- a/sdks/typescript/src/apache_beam/examples/wordcount.ts
+++ b/sdks/typescript/src/apache_beam/examples/wordcount.ts
@@ -53,7 +53,7 @@ async function main() {
         "And the earth was without form, and void; and darkness was upon the face of the deep.",
         "And the Spirit of God moved upon the face of the waters.",
         "And God said, Let there be light: and there was light.",
-      ])
+      ]),
     );
 
     lines.apply(wordCount).map(console.log);
diff --git a/sdks/typescript/src/apache_beam/examples/wordcount_sql.ts b/sdks/typescript/src/apache_beam/examples/wordcount_sql.ts
index 2ea4204886c2..5c5cc575d74d 100644
--- a/sdks/typescript/src/apache_beam/examples/wordcount_sql.ts
+++ b/sdks/typescript/src/apache_beam/examples/wordcount_sql.ts
@@ -39,8 +39,8 @@ async function main() {
       .apply(beam.withRowCoder({ word: "str" }))
       .applyAsync(
         sqlTransform(
-          "SELECT word, count(*) as c from PCOLLECTION group by word"
-        )
+          "SELECT word, count(*) as c from PCOLLECTION group by word",
+        ),
       );
 
     filtered.map(console.log);
diff --git a/sdks/typescript/src/apache_beam/examples/wordcount_textio.ts b/sdks/typescript/src/apache_beam/examples/wordcount_textio.ts
index f618effea1c3..0f8144b95723 100644
--- a/sdks/typescript/src/apache_beam/examples/wordcount_textio.ts
+++ b/sdks/typescript/src/apache_beam/examples/wordcount_textio.ts
@@ -38,7 +38,7 @@ async function main() {
   // python apache_beam/runners/portability/local_job_service_main.py --port 3333
   await new PortableRunner("localhost:3333").run(async (root) => {
     const lines = await root.applyAsync(
-      textio.readFromText("gs://dataflow-samples/shakespeare/kinglear.txt")
+      textio.readFromText("gs://dataflow-samples/shakespeare/kinglear.txt"),
     );
 
     lines.apply(wordCount).map(console.log);
diff --git a/sdks/typescript/src/apache_beam/internal/environments.ts b/sdks/typescript/src/apache_beam/internal/environments.ts
index 4e45800839ea..fa70e80a3e98 100644
--- a/sdks/typescript/src/apache_beam/internal/environments.ts
+++ b/sdks/typescript/src/apache_beam/internal/environments.ts
@@ -40,7 +40,7 @@ export function jsEnvironment(
   urn: string,
   payload: Uint8Array,
   resourceHints: { [key: string]: Uint8Array } = {},
-  artifacts: runnerApi.ArtifactInformation[] = []
+  artifacts: runnerApi.ArtifactInformation[] = [],
 ): runnerApi.Environment {
   return {
     urn: urn,
@@ -55,7 +55,7 @@ export function jsEnvironment(
 function asNewEnvironment(
   env: runnerApi.Environment,
   urn: string,
-  payload: Uint8Array
+  payload: Uint8Array,
 ) {
   return {
     urn: urn,
@@ -69,7 +69,7 @@ function asNewEnvironment(
 
 export function asExternalEnvironment(
   env: runnerApi.Environment,
-  address: string
+  address: string,
 ) {
   return asNewEnvironment(
     env,
@@ -80,17 +80,17 @@ export function asExternalEnvironment(
         authentication: null!,
       },
       params: {},
-    })
+    }),
   );
 }
 
 export function asDockerEnvironment(
   env: runnerApi.Environment,
-  containerImage: string
+  containerImage: string,
 ) {
   return asNewEnvironment(
     env,
     "beam:env:docker:v1",
-    runnerApi.DockerPayload.toBinary({ containerImage: containerImage })
+    runnerApi.DockerPayload.toBinary({ containerImage: containerImage }),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/internal/pipeline.ts b/sdks/typescript/src/apache_beam/internal/pipeline.ts
index fc33fdb25072..6d5878fdaecb 100644
--- a/sdks/typescript/src/apache_beam/internal/pipeline.ts
+++ b/sdks/typescript/src/apache_beam/internal/pipeline.ts
@@ -39,7 +39,10 @@ export class PipelineContext {
 
   private coders: { [key: string]: Coder<any> } = {};
 
-  constructor(public components: Components, private componentPrefix: string) {}
+  constructor(
+    public components: Components,
+    private componentPrefix: string,
+  ) {}
 
   getCoder<T>(coderId: string): Coder<T> {
     const this_ = this;
@@ -51,7 +54,7 @@ export class PipelineContext {
       this.coders[coderId] = globalCoderRegistry().getCoder(
         coderProto.spec!.urn,
         coderProto.spec!.payload,
-        ...components
+        ...components,
       );
     }
     return this.coders[coderId];
@@ -61,7 +64,7 @@ export class PipelineContext {
     return this.getOrAssign(
       this.components.coders,
       coder.toProto(this),
-      "coder"
+      "coder",
     );
   }
 
@@ -82,14 +85,14 @@ export class PipelineContext {
     return this.getOrAssign(
       this.components.windowingStrategies,
       windowing,
-      "windowing"
+      "windowing",
     );
   }
 
   private getOrAssign<T>(
     existing: { [key: string]: T },
     obj: T,
-    prefix: string
+    prefix: string,
   ) {
     for (const [id, other] of Object.entries(existing)) {
       if (equal(other, obj)) {
@@ -134,7 +137,7 @@ export class Pipeline {
 
   preApplyTransform<
     InputT extends pvalue.PValue<any>,
-    OutputT extends pvalue.PValue<any>
+    OutputT extends pvalue.PValue<any>,
   >(transform: AsyncPTransformClass<InputT, OutputT>, input: InputT) {
     const this_ = this;
     const transformId = this.context.createUniqueName("transform");
@@ -153,7 +156,7 @@ export class Pipeline {
     if (this.usedStageNames.has(uniqueName)) {
       throw new Error(
         `Duplicate stage name: "${uniqueName}". ` +
-          "Use beam.withName(...) to give your transform a unique name."
+          "Use beam.withName(...) to give your transform a unique name.",
       );
     }
     this.usedStageNames.add(uniqueName);
@@ -172,11 +175,11 @@ export class Pipeline {
 
   applyTransform<
     InputT extends pvalue.PValue<any>,
-    OutputT extends pvalue.PValue<any>
+    OutputT extends pvalue.PValue<any>,
   >(transform: PTransformClass<InputT, OutputT>, input: InputT) {
     const { id: transformId, proto: transformProto } = this.preApplyTransform(
       transform,
-      input
+      input,
     );
     let result: OutputT;
     try {
@@ -190,11 +193,11 @@ export class Pipeline {
 
   async applyAsyncTransform<
     InputT extends pvalue.PValue<any>,
-    OutputT extends pvalue.PValue<any>
+    OutputT extends pvalue.PValue<any>,
   >(transform: AsyncPTransformClass<InputT, OutputT>, input: InputT) {
     const { id: transformId, proto: transformProto } = this.preApplyTransform(
       transform,
-      input
+      input,
     );
     let result: OutputT;
     try {
@@ -208,26 +211,26 @@ export class Pipeline {
 
   postApplyTransform<
     InputT extends pvalue.PValue<any>,
-    OutputT extends pvalue.PValue<any>
+    OutputT extends pvalue.PValue<any>,
   >(
     transform: AsyncPTransformClass<InputT, OutputT>,
     transformProto: runnerApi.PTransform,
-    result: OutputT
+    result: OutputT,
   ) {
     transformProto.outputs = objectMap(pvalue.flattenPValue(result), (pc) =>
-      pc.getId()
+      pc.getId(),
     );
 
     // Propagate any unset pvalue.PCollection properties.
     const this_ = this;
     const inputProtos = Object.values(transformProto.inputs).map(
-      (id) => this_.proto.components!.pcollections[id]
+      (id) => this_.proto.components!.pcollections[id],
     );
     const inputBoundedness = new Set(
-      inputProtos.map((proto) => proto.isBounded)
+      inputProtos.map((proto) => proto.isBounded),
     );
     const inputWindowings = new Set(
-      inputProtos.map((proto) => proto.windowingStrategyId)
+      inputProtos.map((proto) => proto.windowingStrategyId),
     );
 
     for (const pcId of Object.values(transformProto.outputs)) {
@@ -235,7 +238,7 @@ export class Pipeline {
       if (!pcProto.isBounded) {
         pcProto.isBounded = onlyValueOr(
           inputBoundedness,
-          runnerApi.IsBounded_Enum.BOUNDED
+          runnerApi.IsBounded_Enum.BOUNDED,
         );
       }
       // TODO: (Cleanup) Handle the case of equivalent strategies.
@@ -246,9 +249,9 @@ export class Pipeline {
           (a, b) => {
             return equal(
               this_.proto.components!.windowingStrategies[a],
-              this_.proto.components!.windowingStrategies[b]
+              this_.proto.components!.windowingStrategies[b],
             );
-          }
+          },
         );
       }
     }
@@ -262,11 +265,11 @@ export class Pipeline {
       | runnerApi.WindowingStrategy
       | string
       | undefined = undefined,
-    isBounded: runnerApi.IsBounded_Enum | undefined = undefined
+    isBounded: runnerApi.IsBounded_Enum | undefined = undefined,
   ): pvalue.PCollection<OutputT> {
     return new pvalue.PCollection<OutputT>(
       this,
-      this.createPCollectionIdInternal(coder, windowingStrategy, isBounded)
+      this.createPCollectionIdInternal(coder, windowingStrategy, isBounded),
     );
   }
 
@@ -276,7 +279,7 @@ export class Pipeline {
       | runnerApi.WindowingStrategy
       | string
       | undefined = undefined,
-    isBounded: runnerApi.IsBounded_Enum | undefined = undefined
+    isBounded: runnerApi.IsBounded_Enum | undefined = undefined,
   ): string {
     const pcollId = this.context.createUniqueName("pc");
     let coderId: string;
@@ -292,7 +295,7 @@ export class Pipeline {
       windowingStrategyId = windowingStrategy;
     } else {
       windowingStrategyId = this.context.getWindowingStrategyId(
-        windowingStrategy!
+        windowingStrategy!,
       );
     }
     this.proto!.components!.pcollections[pcollId] = {
@@ -325,7 +328,7 @@ function objectMap(obj, func) {
 function onlyValueOr<T>(
   valueSet: Set<T>,
   defaultValue: T,
-  comparator: (a: T, b: T) => boolean = (a, b) => false
+  comparator: (a: T, b: T) => boolean = (a, b) => false,
 ) {
   if (valueSet.size === 0) {
     return defaultValue;
diff --git a/sdks/typescript/src/apache_beam/internal/serialize.ts b/sdks/typescript/src/apache_beam/internal/serialize.ts
index 3a4bf81140e0..ccc3532d29b9 100644
--- a/sdks/typescript/src/apache_beam/internal/serialize.ts
+++ b/sdks/typescript/src/apache_beam/internal/serialize.ts
@@ -38,11 +38,11 @@ export function serializeFn(obj: unknown): Uint8Array {
     JSON.stringify(
       serialize_closures.serialize(
         obj,
-        serialize_closures.defaultBuiltins.concat(registeredObjects)
+        serialize_closures.defaultBuiltins.concat(registeredObjects),
       ),
       (key, value) =>
-        typeof value === "bigint" ? `${BIGINT_PREFIX}${value}` : value
-    )
+        typeof value === "bigint" ? `${BIGINT_PREFIX}${value}` : value,
+    ),
   );
 }
 
@@ -51,8 +51,8 @@ export function deserializeFn(s: Uint8Array): any {
     JSON.parse(new TextDecoder().decode(s), (key, value) =>
       typeof value === "string" && value.startsWith(BIGINT_PREFIX)
         ? BigInt(value.substr(BIGINT_PREFIX.length))
-        : value
+        : value,
     ),
-    serialize_closures.defaultBuiltins.concat(registeredObjects)
+    serialize_closures.defaultBuiltins.concat(registeredObjects),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/io/avroio.ts b/sdks/typescript/src/apache_beam/io/avroio.ts
index b6ebf45cf285..a638fde18184 100644
--- a/sdks/typescript/src/apache_beam/io/avroio.ts
+++ b/sdks/typescript/src/apache_beam/io/avroio.ts
@@ -25,31 +25,31 @@ import { withCoderInternal } from "../transforms/internal";
 export function readFromAvro<T>(
   filePattern: string,
   // TODO: Allow schema to be inferred.
-  options: { schema: Schema }
+  options: { schema: Schema },
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<T>> {
   return schemaio<beam.Root, beam.PCollection<T>>(
     "readFromTable",
     "beam:transform:org.apache.beam:schemaio_avro_read:v1",
-    { location: filePattern, schema: options.schema }
+    { location: filePattern, schema: options.schema },
   );
 }
 
 export function writeToAvro<T>(filePath: string, options: { schema: Schema }) {
   return async function writeToAvro(
-    pcoll: beam.PCollection<Object>
+    pcoll: beam.PCollection<Object>,
   ): Promise<{}> {
     // TODO: Allow schema to be inferred.
     if (options.schema) {
       pcoll = pcoll.apply(
-        withCoderInternal(RowCoder.fromSchema(options.schema))
+        withCoderInternal(RowCoder.fromSchema(options.schema)),
       );
     }
     return pcoll.applyAsync(
       schemaio<beam.PCollection<T>, {}>(
         "writeToAvro",
         "beam:transform:org.apache.beam:schemaio_avro_write:v1",
-        { location: filePath, schema: options.schema }
-      )
+        { location: filePath, schema: options.schema },
+      ),
     );
   };
 }
diff --git a/sdks/typescript/src/apache_beam/io/bigqueryio.ts b/sdks/typescript/src/apache_beam/io/bigqueryio.ts
index ff4800cb0bc9..886728801aef 100644
--- a/sdks/typescript/src/apache_beam/io/bigqueryio.ts
+++ b/sdks/typescript/src/apache_beam/io/bigqueryio.ts
@@ -32,19 +32,19 @@ const bigqueryIOConfigSchema = RowCoder.inferSchemaOfJSON({
 export function readFromBigQuery<T>(
   options:
     | { table: string; schema?: Schema }
-    | { query: string; schema?: Schema }
+    | { query: string; schema?: Schema },
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<T>> {
   return schemaio<beam.Root, beam.PCollection<T>>(
     "readFromBigQuery",
     "beam:transform:org.apache.beam:schemaio_bigquery_read:v1",
     options,
-    bigqueryIOConfigSchema
+    bigqueryIOConfigSchema,
   );
 }
 
 export function writeToBigQuery<T>(
   table: string,
-  options: { createDisposition?: "Never" | "IfNeeded" } = {}
+  options: { createDisposition?: "Never" | "IfNeeded" } = {},
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<T>> {
   if (options.createDisposition == undefined) {
     options.createDisposition = "IfNeeded";
@@ -53,6 +53,6 @@ export function writeToBigQuery<T>(
     "writeToBigquery",
     "beam:transform:org.apache.beam:schemaio_bigquery_write:v1",
     { table, createDisposition: options.createDisposition },
-    bigqueryIOConfigSchema
+    bigqueryIOConfigSchema,
   );
 }
diff --git a/sdks/typescript/src/apache_beam/io/kafka.ts b/sdks/typescript/src/apache_beam/io/kafka.ts
index 8fd717ec214d..99600d75e6c6 100644
--- a/sdks/typescript/src/apache_beam/io/kafka.ts
+++ b/sdks/typescript/src/apache_beam/io/kafka.ts
@@ -48,28 +48,28 @@ const defaultReadFromKafkaOptions = {
 export function readFromKafka<T>(
   consumerConfig: { [key: string]: string }, // TODO: Or a map?
   topics: string[],
-  options: ReadFromKafkaOptions = {}
+  options: ReadFromKafkaOptions = {},
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<T>> {
   return readFromKafkaMaybeWithMetadata(
     "readFromKafkaWithMetadata",
     "beam:transform:org.apache.beam:kafka_read_without_metadata:v1",
     consumerConfig,
     topics,
-    options
+    options,
   );
 }
 
 export function readFromKafkaWithMetadata<T>(
   consumerConfig: { [key: string]: string }, // TODO: Or a map?
   topics: string[],
-  options: ReadFromKafkaOptions = {}
+  options: ReadFromKafkaOptions = {},
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<T>> {
   return readFromKafkaMaybeWithMetadata<T>(
     "readFromKafkaWithMetadata",
     "beam:transform:org.apache.beam:kafka_read_with_metadata:v1",
     consumerConfig,
     topics,
-    options
+    options,
   );
 }
 
@@ -78,7 +78,7 @@ function readFromKafkaMaybeWithMetadata<T>(
   urn: string,
   consumerConfig: { [key: string]: string }, // TODO: Or a map?
   topics: string[],
-  options: ReadFromKafkaOptions = {}
+  options: ReadFromKafkaOptions = {},
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<T>> {
   return beam.withName(
     name,
@@ -89,8 +89,8 @@ function readFromKafkaMaybeWithMetadata<T>(
         consumerConfig,
         ...camelToSnakeOptions({ ...defaultReadFromKafkaOptions, ...options }),
       },
-      serviceProviderFromJavaGradleTarget(KAFKA_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(KAFKA_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
 
@@ -107,7 +107,7 @@ const defaultWriteToKafkaOptions = {
 export function writeToKafka<K = Uint8Array, V = Uint8Array>(
   producerConfig: { [key: string]: string }, // TODO: Or a map?
   topics: string[],
-  options: WriteToKafkaOptions = {}
+  options: WriteToKafkaOptions = {},
 ): beam.AsyncPTransform<beam.PCollection<{ key: K; value: V }>, {}> {
   return beam.withName(
     "writeToKafka",
@@ -118,7 +118,7 @@ export function writeToKafka<K = Uint8Array, V = Uint8Array>(
         producerConfig,
         ...camelToSnakeOptions({ ...defaultWriteToKafkaOptions, ...options }),
       },
-      serviceProviderFromJavaGradleTarget(KAFKA_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(KAFKA_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/io/parquetio.ts b/sdks/typescript/src/apache_beam/io/parquetio.ts
index e7c7f7dc66e0..ade4d289ecdd 100644
--- a/sdks/typescript/src/apache_beam/io/parquetio.ts
+++ b/sdks/typescript/src/apache_beam/io/parquetio.ts
@@ -30,7 +30,7 @@ export function readFromParquet(
   filePattern: string,
   options: {
     columns?: string[];
-  } = {}
+  } = {},
 ): (root: beam.Root) => Promise<beam.PCollection<any>> {
   return async function readFromParquet(root: beam.Root) {
     return root.applyAsync(
@@ -38,21 +38,21 @@ export function readFromParquet(
         path: filePattern,
         format: "parquet",
         ...camelToSnakeOptions(options),
-      })
+      }),
     );
   };
 }
 
 export function writeToParquet(
   filePathPrefix: string,
-  options: { schema?: Schema } = {}
+  options: { schema?: Schema } = {},
 ): (
-  toWrite: beam.PCollection<Object>
+  toWrite: beam.PCollection<Object>,
 ) => Promise<{ filesWritten: beam.PCollection<string> }> {
   return async function writeToJson(toWrite: beam.PCollection<Object>) {
     if (options.schema) {
       toWrite = toWrite.apply(
-        withCoderInternal(RowCoder.fromSchema(options.schema))
+        withCoderInternal(RowCoder.fromSchema(options.schema)),
       );
       delete options.schema;
     }
@@ -62,7 +62,7 @@ export function writeToParquet(
           path: filePathPrefix,
           format: "parquet",
           ...camelToSnakeOptions(options),
-        })
+        }),
       ),
     };
   };
diff --git a/sdks/typescript/src/apache_beam/io/pubsub.ts b/sdks/typescript/src/apache_beam/io/pubsub.ts
index c5513fe41718..c7ae7ee5462d 100644
--- a/sdks/typescript/src/apache_beam/io/pubsub.ts
+++ b/sdks/typescript/src/apache_beam/io/pubsub.ts
@@ -53,11 +53,11 @@ type ReadOptions =
 
 // TODO: Schema-producing variants.
 export function readFromPubSub(
-  options: ReadOptions
+  options: ReadOptions,
 ): AsyncPTransform<beam.Root, beam.PCollection<Uint8Array>> {
   if (options.topic && options.subscription) {
     throw new TypeError(
-      "Exactly one of topic or subscription must be provided."
+      "Exactly one of topic or subscription must be provided.",
     );
   }
   return withName(
@@ -65,17 +65,17 @@ export function readFromPubSub(
     external.rawExternalTransform<beam.Root, beam.PCollection<Uint8Array>>(
       "beam:transform:org.apache.beam:pubsub_read:v1",
       camelToSnakeOptions(options),
-      serviceProviderFromJavaGradleTarget(PUBSUB_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(PUBSUB_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
 
 function readFromPubSubWithAttributesRaw(
-  options: ReadOptions
+  options: ReadOptions,
 ): AsyncPTransform<beam.Root, beam.PCollection<Uint8Array>> {
   if (options.topic && options.subscription) {
     throw new TypeError(
-      "Exactly one of topic or subscription must be provided."
+      "Exactly one of topic or subscription must be provided.",
     );
   }
   return withName(
@@ -83,13 +83,13 @@ function readFromPubSubWithAttributesRaw(
     external.rawExternalTransform<beam.Root, beam.PCollection<Uint8Array>>(
       "beam:transform:org.apache.beam:pubsub_read:v1",
       { needsAttributes: true, ...camelToSnakeOptions(options) },
-      serviceProviderFromJavaGradleTarget(PUBSUB_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(PUBSUB_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
 
 export function readFromPubSubWithAttributes(
-  options: ReadOptions
+  options: ReadOptions,
 ): AsyncPTransform<
   beam.Root,
   beam.PCollection<PubSub.protos.google.pubsub.v1.PubsubMessage>
@@ -98,7 +98,7 @@ export function readFromPubSubWithAttributes(
     return (
       await root.applyAsync(readFromPubSubWithAttributesRaw(options))
     ).map((encoded) =>
-      PubSub.protos.google.pubsub.v1.PubsubMessage.decode(encoded)
+      PubSub.protos.google.pubsub.v1.PubsubMessage.decode(encoded),
     );
   };
 }
@@ -107,15 +107,15 @@ type WriteOptions = { idAttribute?: string; timestampAttribute?: string };
 
 function writeToPubSubRaw(
   topic: string,
-  options: WriteOptions = {}
+  options: WriteOptions = {},
 ): AsyncPTransform<beam.PCollection<Uint8Array>, {}> {
   return withName(
     "writeToPubSubRaw",
     external.rawExternalTransform<beam.PCollection<Uint8Array>, {}>(
       "beam:transform:org.apache.beam:pubsub_write:v1",
       { topic, ...camelToSnakeOptions(options) },
-      serviceProviderFromJavaGradleTarget(PUBSUB_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(PUBSUB_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
 
@@ -123,7 +123,7 @@ export function writeToPubSub(topic: string, options: WriteOptions = {}) {
   return async function writeToPubSub(dataPColl: beam.PCollection<Uint8Array>) {
     return dataPColl //
       .map((data) =>
-        PubSub.protos.google.pubsub.v1.PubsubMessage.encode({ data }).finish()
+        PubSub.protos.google.pubsub.v1.PubsubMessage.encode({ data }).finish(),
       )
       .apply(internal.withCoderInternal(new BytesCoder()))
       .applyAsync(writeToPubSubRaw(topic, options));
diff --git a/sdks/typescript/src/apache_beam/io/pubsublite.ts b/sdks/typescript/src/apache_beam/io/pubsublite.ts
index 2798d44079bf..510b39b9bf9a 100644
--- a/sdks/typescript/src/apache_beam/io/pubsublite.ts
+++ b/sdks/typescript/src/apache_beam/io/pubsublite.ts
@@ -28,28 +28,28 @@ const PUBSUBLITE_EXPANSION_GRADLE_TARGET =
 // TODO: Schema-producing variants.
 export function readFromPubSubLiteRaw(
   subscriptionPath: string,
-  options: { minBundleTimeout?: number; deduplicate?: boolean } = {}
+  options: { minBundleTimeout?: number; deduplicate?: boolean } = {},
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<Uint8Array>> {
   return beam.withName(
     "readFromPubSubLiteRaw",
     external.rawExternalTransform<beam.Root, beam.PCollection<Uint8Array>>(
       "beam:transform:org.apache.beam:pubsublite_read:v1",
       { subscription_path: subscriptionPath, ...camelToSnakeOptions(options) },
-      serviceProviderFromJavaGradleTarget(PUBSUBLITE_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(PUBSUBLITE_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
 
 export function writeToPubSubLiteRaw(
   topicPath: string,
-  options: { addUuids?: boolean } = {}
+  options: { addUuids?: boolean } = {},
 ): beam.AsyncPTransform<beam.PCollection<Uint8Array>, {}> {
   return beam.withName(
     "writeToPubSubLiteRaw",
     external.rawExternalTransform<beam.PCollection<Uint8Array>, {}>(
       "beam:transform:org.apache.beam:pubsublite_write:v1",
       { topic_path: topicPath, ...camelToSnakeOptions(options) },
-      serviceProviderFromJavaGradleTarget(PUBSUBLITE_EXPANSION_GRADLE_TARGET)
-    )
+      serviceProviderFromJavaGradleTarget(PUBSUBLITE_EXPANSION_GRADLE_TARGET),
+    ),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/io/schemaio.ts b/sdks/typescript/src/apache_beam/io/schemaio.ts
index c9f666b3a772..20cb7f8abca4 100644
--- a/sdks/typescript/src/apache_beam/io/schemaio.ts
+++ b/sdks/typescript/src/apache_beam/io/schemaio.ts
@@ -25,12 +25,12 @@ import * as protobufjs from "protobufjs";
 
 export function schemaio<
   InputT extends beam.PValue<any>,
-  OutputT extends beam.PValue<any>
+  OutputT extends beam.PValue<any>,
 >(
   name,
   urn,
   config,
-  configSchema: Schema | undefined = undefined
+  configSchema: Schema | undefined = undefined,
 ): beam.AsyncPTransform<InputT, OutputT> {
   // Location is separate for historical reasons.
   let maybeLocation: { location?: string } = {};
@@ -58,8 +58,8 @@ export function schemaio<
       urn,
       { config: encodedConfig, ...maybeLocation, ...maybeSchema },
       serviceProviderFromJavaGradleTarget(
-        "sdks:java:io:google-cloud-platform:expansion-service:shadowJar"
-      )
-    )
+        "sdks:java:io:google-cloud-platform:expansion-service:shadowJar",
+      ),
+    ),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/io/textio.ts b/sdks/typescript/src/apache_beam/io/textio.ts
index 7dedbcf26989..fb2cb41ec679 100644
--- a/sdks/typescript/src/apache_beam/io/textio.ts
+++ b/sdks/typescript/src/apache_beam/io/textio.ts
@@ -27,7 +27,7 @@ import { Schema } from "../proto/schema";
 import { RowCoder } from "../coders/row_coder";
 
 export function readFromText(
-  filePattern: string
+  filePattern: string,
 ): beam.AsyncPTransform<beam.Root, beam.PCollection<string>> {
   return async function readFromText(root: beam.Root) {
     return root.applyAsync(
@@ -35,8 +35,8 @@ export function readFromText(
         "apache_beam.io.ReadFromText",
         {
           file_pattern: filePattern,
-        }
-      )
+        },
+      ),
     );
   };
 }
@@ -50,9 +50,9 @@ export function writeToText(
     shardNameTemplate?: string;
     header?: string;
     footer?: string;
-  } = {}
+  } = {},
 ): (
-  toWrite: beam.PCollection<string>
+  toWrite: beam.PCollection<string>,
 ) => Promise<{ filesWritten: beam.PCollection<string> }> {
   return async function writeToText(pcoll: beam.PCollection<string>) {
     return {
@@ -63,7 +63,7 @@ export function writeToText(
           pythonTransform("apache_beam.io.WriteToText", {
             file_path_prefix: filePathPrefix,
             ...camelToSnakeOptions(options),
-          })
+          }),
         ),
     };
   };
@@ -71,7 +71,7 @@ export function writeToText(
 
 export function readFromCsv(
   filePattern: string,
-  options: {} = {}
+  options: {} = {},
 ): (root: beam.Root) => Promise<beam.PCollection<any>> {
   return async function readFromCsv(root: beam.Root) {
     return root.applyAsync(
@@ -79,7 +79,7 @@ export function readFromCsv(
         path: filePattern,
         format: "csv",
         ...camelToSnakeOptions(options),
-      })
+      }),
     );
   };
 }
@@ -87,9 +87,9 @@ export function readFromCsv(
 export function writeToCsv(
   filePathPrefix: string,
   schema: Schema | undefined = undefined,
-  options: {} = {}
+  options: {} = {},
 ): (
-  toWrite: beam.PCollection<Object>
+  toWrite: beam.PCollection<Object>,
 ) => Promise<{ filesWritten: beam.PCollection<string> }> {
   return async function writeToCsv(toWrite: beam.PCollection<Object>) {
     if (schema != undefined) {
@@ -102,7 +102,7 @@ export function writeToCsv(
           format: "csv",
           index: false,
           ...camelToSnakeOptions(options),
-        })
+        }),
       ),
     };
   };
@@ -110,7 +110,7 @@ export function writeToCsv(
 
 export function readFromJson(
   filePattern: string,
-  options: {} = {}
+  options: {} = {},
 ): (root: beam.Root) => Promise<beam.PCollection<any>> {
   return async function readFromJson(root: beam.Root) {
     return root.applyAsync(
@@ -120,7 +120,7 @@ export function readFromJson(
         orient: "records",
         lines: true,
         ...camelToSnakeOptions(options),
-      })
+      }),
     );
   };
 }
@@ -128,9 +128,9 @@ export function readFromJson(
 export function writeToJson(
   filePathPrefix: string,
   schema: Schema | undefined = undefined,
-  options: {} = {}
+  options: {} = {},
 ): (
-  toWrite: beam.PCollection<Object>
+  toWrite: beam.PCollection<Object>,
 ) => Promise<{ filesWritten: beam.PCollection<string> }> {
   return async function writeToJson(toWrite: beam.PCollection<Object>) {
     if (schema != undefined) {
@@ -144,7 +144,7 @@ export function writeToJson(
           orient: "records",
           lines: true,
           ...camelToSnakeOptions(options),
-        })
+        }),
       ),
     };
   };
diff --git a/sdks/typescript/src/apache_beam/pvalue.ts b/sdks/typescript/src/apache_beam/pvalue.ts
index e3b7972f6654..6a6c34cdb3b6 100644
--- a/sdks/typescript/src/apache_beam/pvalue.ts
+++ b/sdks/typescript/src/apache_beam/pvalue.ts
@@ -50,7 +50,7 @@ export class Root {
   }
 
   async applyAsync<OutputT extends PValue<any>>(
-    transform: AsyncPTransform<Root, OutputT>
+    transform: AsyncPTransform<Root, OutputT>,
   ) {
     if (!(transform instanceof AsyncPTransformClass)) {
       transform = new AsyncPTransformClassFromCallable(transform);
@@ -88,7 +88,7 @@ export class PCollection<T> {
   }
 
   apply<OutputT extends PValue<any>>(
-    transform: PTransform<PCollection<T>, OutputT>
+    transform: PTransform<PCollection<T>, OutputT>,
   ) {
     if (!(transform instanceof PTransformClass)) {
       transform = new PTransformClassFromCallable(transform);
@@ -97,7 +97,7 @@ export class PCollection<T> {
   }
 
   applyAsync<OutputT extends PValue<any>>(
-    transform: AsyncPTransform<PCollection<T>, OutputT>
+    transform: AsyncPTransform<PCollection<T>, OutputT>,
   ) {
     if (!(transform instanceof AsyncPTransformClass)) {
       transform = new AsyncPTransformClassFromCallable(transform);
@@ -109,7 +109,7 @@ export class PCollection<T> {
     fn:
       | (ContextT extends undefined ? (element: T) => OutputT : never)
       | ((element: T, context: ContextT) => OutputT),
-    context: ContextT = undefined!
+    context: ContextT = undefined!,
   ): PCollection<OutputT> {
     if (extractContext(fn)) {
       context = { ...extractContext(fn), ...context };
@@ -127,9 +127,9 @@ export class PCollection<T> {
                 : fn(element, context);
             },
           },
-          context
-        )
-      )
+          context,
+        ),
+      ),
     );
   }
 
@@ -137,7 +137,7 @@ export class PCollection<T> {
     fn:
       | (ContextT extends undefined ? (element: T) => Iterable<OutputT> : never)
       | ((element: T, context: ContextT) => Iterable<OutputT>),
-    context: ContextT = undefined!
+    context: ContextT = undefined!,
   ): PCollection<OutputT> {
     if (extractContext(fn)) {
       context = { ...extractContext(fn), ...context };
@@ -155,9 +155,9 @@ export class PCollection<T> {
                 : fn(element, context);
             },
           },
-          context
-        )
-      )
+          context,
+        ),
+      ),
     );
   }
 
@@ -184,7 +184,7 @@ export type PValue<T> =
  */
 export function flattenPValue<T>(
   pValue: PValue<T>,
-  prefix: string = ""
+  prefix: string = "",
 ): { [key: string]: PCollection<T> } {
   const result: { [key: string]: PCollection<any> } = {};
   if (pValue === null || pValue === undefined) {
@@ -231,7 +231,7 @@ class PValueWrapper<T extends PValue<any>> {
 
   apply<O extends PValue<any>>(
     transform: PTransform<T, O>,
-    root: Root | null = null
+    root: Root | null = null,
   ) {
     if (!(transform instanceof PTransformClass)) {
       transform = new PTransformClassFromCallable(transform);
@@ -241,14 +241,14 @@ class PValueWrapper<T extends PValue<any>> {
 
   async applyAsync<O extends PValue<any>>(
     transform: AsyncPTransform<T, O>,
-    root: Root | null = null
+    root: Root | null = null,
   ) {
     if (!(transform instanceof AsyncPTransformClass)) {
       transform = new AsyncPTransformClassFromCallable(transform);
     }
     return await this.pipeline(root).applyAsyncTransform(
       transform,
-      this.pvalue
+      this.pvalue,
     );
   }
 
@@ -264,20 +264,20 @@ class PValueWrapper<T extends PValue<any>> {
 
 class PTransformClassFromCallable<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > extends PTransformClass<InputT, OutputT> {
   expander: (
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) => OutputT;
 
   constructor(
     expander: (
       input: InputT,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
-    ) => OutputT
+      transformProto: runnerApi.PTransform,
+    ) => OutputT,
   ) {
     super(extractName(expander));
     this.expander = expander;
@@ -286,7 +286,7 @@ class PTransformClassFromCallable<
   expandInternal(
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     return this.expander(input, pipeline, transformProto);
   }
@@ -294,20 +294,20 @@ class PTransformClassFromCallable<
 
 class AsyncPTransformClassFromCallable<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > extends AsyncPTransformClass<InputT, OutputT> {
   expander: (
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) => Promise<OutputT>;
 
   constructor(
     expander: (
       input: InputT,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
-    ) => Promise<OutputT>
+      transformProto: runnerApi.PTransform,
+    ) => Promise<OutputT>,
   ) {
     super(extractName(expander));
     this.expander = expander;
@@ -316,7 +316,7 @@ class AsyncPTransformClassFromCallable<
   async expandInternalAsync(
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     return this.expander(input, pipeline, transformProto);
   }
diff --git a/sdks/typescript/src/apache_beam/runners/artifacts.ts b/sdks/typescript/src/apache_beam/runners/artifacts.ts
index f5678621ba56..3777dacada7b 100644
--- a/sdks/typescript/src/apache_beam/runners/artifacts.ts
+++ b/sdks/typescript/src/apache_beam/runners/artifacts.ts
@@ -33,7 +33,7 @@ const defaultArtifactDir = path.join(
   os.homedir(),
   ".apache_beam",
   "cache",
-  "artifacts"
+  "artifacts",
 );
 
 /**
@@ -43,7 +43,7 @@ const defaultArtifactDir = path.join(
 export async function* resolveArtifacts(
   client: IArtifactRetrievalServiceClient,
   artifacts: Iterable<runnerApi.ArtifactInformation>,
-  localDir: string = defaultArtifactDir
+  localDir: string = defaultArtifactDir,
 ): AsyncGenerator<runnerApi.ArtifactInformation, void, unknown> {
   const resolved = await client.resolveArtifacts({
     artifacts: Array.from(artifacts),
@@ -51,11 +51,11 @@ export async function* resolveArtifacts(
   }).response;
 
   async function storeArtifact(
-    artifact: runnerApi.ArtifactInformation
+    artifact: runnerApi.ArtifactInformation,
   ): Promise<runnerApi.ArtifactInformation> {
     if (artifact.typeUrn === "beam:artifact:type:file:v1") {
       const payload = runnerApi.ArtifactFilePayload.fromBinary(
-        artifact.typePayload
+        artifact.typePayload,
       );
       // As we're storing artifacts by hash, we can safely re-use if we've
       // ever seen this before.
@@ -116,7 +116,7 @@ export async function* resolveArtifacts(
 export async function offerArtifacts(
   client: IArtifactStagingServiceClient,
   stagingToken: string,
-  rootDir: string = defaultArtifactDir
+  rootDir: string = defaultArtifactDir,
 ) {
   const call = client.reverseArtifactRetrievalService();
   call.responses.onMessage(async (msg) => {
@@ -138,7 +138,7 @@ export async function offerArtifacts(
         switch (msg.request.getArtifact.artifact!.typeUrn) {
           case "beam:artifact:type:file:v1":
             const payload = runnerApi.ArtifactFilePayload.fromBinary(
-              msg.request.getArtifact.artifact!.typePayload
+              msg.request.getArtifact.artifact!.typePayload,
             );
             const filePath = path.normalize(payload.path);
             if (!filePath.startsWith(rootDir)) {
@@ -146,7 +146,7 @@ export async function offerArtifacts(
                 "Refusing to serve " +
                   filePath +
                   " as it is not under " +
-                  rootDir
+                  rootDir,
               );
             }
             const handle = fs.createReadStream(filePath);
@@ -173,7 +173,7 @@ export async function offerArtifacts(
           default:
             throw new Error(
               "Unknown artifact type " +
-                msg.request.getArtifact.artifact!.typeUrn
+                msg.request.getArtifact.artifact!.typeUrn,
             );
         }
         break;
@@ -200,6 +200,6 @@ function tempFile(dir) {
       crypto
         .randomBytes(16)
         .toString("base64")
-        .replace(/[^a-zA-Z0-9]/g, "_")
+        .replace(/[^a-zA-Z0-9]/g, "_"),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/runners/dataflow.ts b/sdks/typescript/src/apache_beam/runners/dataflow.ts
index e7da1f7ada51..5e1c052148fa 100644
--- a/sdks/typescript/src/apache_beam/runners/dataflow.ts
+++ b/sdks/typescript/src/apache_beam/runners/dataflow.ts
@@ -30,19 +30,26 @@ export function dataflowRunner(runnerOptions: {
   return new (class extends Runner {
     async runPipeline(
       pipeline: Pipeline,
-      options: Object = {}
+      options: Object = {},
     ): Promise<PipelineResult> {
       var augmentedOptions = { experiments: [] as string[], ...options };
       augmentedOptions.experiments.push("use_runner_v2");
       augmentedOptions.experiments.push("use_portable_job_submission");
       augmentedOptions.experiments.push("use_sibling_sdk_workers");
-      return new PortableRunner(
+      const service = PythonService.forModule(
+        "apache_beam.runners.dataflow.dataflow_job_service",
+        ["--port", "{{PORT}}"],
+      );
+      const result = new PortableRunner(
         runnerOptions as any,
-        PythonService.forModule(
-          "apache_beam.runners.dataflow.dataflow_job_service",
-          ["--port", "{{PORT}}"]
-        )
+        service,
       ).runPipeline(pipeline, augmentedOptions);
+      result.then((res) => {
+        res.waitUntilFinish().then((_state) => {
+          service.stop();
+        });
+      });
+      return result;
     }
   })();
 }
diff --git a/sdks/typescript/src/apache_beam/runners/direct_runner.ts b/sdks/typescript/src/apache_beam/runners/direct_runner.ts
index 999026a884ee..d616e2d4f129 100644
--- a/sdks/typescript/src/apache_beam/runners/direct_runner.ts
+++ b/sdks/typescript/src/apache_beam/runners/direct_runner.ts
@@ -85,7 +85,7 @@ class DirectRunner extends Runner {
     }
 
     for (const windowing of Object.values(
-      proto.components!.windowingStrategies
+      proto.components!.windowingStrategies,
     )) {
       if (
         ![
@@ -122,7 +122,7 @@ class DirectRunner extends Runner {
         descriptor,
         null!,
         new state.CachingStateProvider(stateProvider),
-        [impulse.urn]
+        [impulse.urn],
       );
       await processor.process("bundle_id");
 
@@ -134,7 +134,7 @@ class DirectRunner extends Runner {
           const shortIdCache = new metrics.MetricsShortIdCache();
           const monitoringData = processor.monitoringData(shortIdCache);
           return Array.from(monitoringData.entries()).map(([id, payload]) =>
-            shortIdCache.asMonitoringInfo(id, payload)
+            shortIdCache.asMonitoringInfo(id, payload),
           );
         }
       })();
@@ -151,10 +151,10 @@ class DirectImpulseOperator implements operators.IOperator {
   constructor(
     public transformId: string,
     transform: PTransform,
-    context: operators.OperatorContext
+    context: operators.OperatorContext,
   ) {
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
   }
 
@@ -189,17 +189,17 @@ class DirectGbkOperator implements operators.IOperator {
   constructor(
     public transformId: string,
     transform: PTransform,
-    context: operators.OperatorContext
+    context: operators.OperatorContext,
   ) {
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
     const inputPc =
       context.descriptor.pcollections[
         onlyElement(Object.values(transform.inputs))
       ];
     this.keyCoder = context.pipelineContext.getCoder(
-      context.descriptor.coders[inputPc.coderId].componentCoderIds[0]
+      context.descriptor.coders[inputPc.coderId].componentCoderIds[0],
     );
     const windowingStrategy =
       context.descriptor.windowingStrategies[inputPc.windowingStrategyId];
@@ -212,11 +212,11 @@ class DirectGbkOperator implements operators.IOperator {
       windowingStrategy.outputTime !== runnerApi.OutputTime_Enum.END_OF_WINDOW
     ) {
       throw new Error(
-        "Unsupported windowing output time: " + windowingStrategy
+        "Unsupported windowing output time: " + windowingStrategy,
       );
     }
     this.windowCoder = context.pipelineContext.getCoder(
-      windowingStrategy.windowCoderId
+      windowingStrategy.windowCoderId,
     );
   }
 
@@ -245,7 +245,7 @@ class DirectGbkOperator implements operators.IOperator {
       const encodedKey = parts[1];
       const window = operators.decodeFromBase64(
         encodedWindow,
-        this.windowCoder
+        this.windowCoder,
       );
       const maybePromise = this.receiver.receive({
         value: {
@@ -347,13 +347,13 @@ function rewriteSideInputs(p: runnerApi.Pipeline, pipelineStateRef: string) {
         transform.inputs[side] = sideCopyId;
         const controlPCollId = uniqueName(
           pcolls,
-          sidePCollId + "-" + side + "-control"
+          sidePCollId + "-" + side + "-control",
         );
         pcolls[controlPCollId] = pcolls[transform.inputs[mainPCollTag]];
         bufferInputs[side] = controlPCollId;
         const collectTransformId = uniqueName(
           transforms,
-          transformId + "-" + side + "-collect"
+          transformId + "-" + side + "-collect",
         );
         transforms[collectTransformId] = runnerApi.PTransform.create({
           spec: {
@@ -402,17 +402,17 @@ class CollectSideOperator implements operators.IOperator {
   constructor(
     public transformId: string,
     transform: PTransform,
-    context: operators.OperatorContext
+    context: operators.OperatorContext,
   ) {
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
     const payload = deserializeFn(transform.spec!.payload!);
     this.parDoTransformId = payload.transformId;
     this.accessPattern = payload.accessPattern;
     this.sideInputId = payload.sideInputId;
     this.stateProvider = DirectRunner.inMemoryStatesRefs.get(
-      payload.pipelineStateRef
+      payload.pipelineStateRef,
     )!;
 
     const inputPc =
@@ -423,7 +423,7 @@ class CollectSideOperator implements operators.IOperator {
     const windowingStrategy =
       context.descriptor.windowingStrategies[inputPc.windowingStrategyId];
     this.windowCoder = context.pipelineContext.getCoder(
-      windowingStrategy.windowCoderId
+      windowingStrategy.windowCoderId,
     );
   }
 
@@ -435,7 +435,7 @@ class CollectSideOperator implements operators.IOperator {
       this.elementCoder.encode(
         wvalue.value,
         writer,
-        CoderContext.needsDelimiters
+        CoderContext.needsDelimiters,
       );
       const encodedElement = writer.finish();
       this.stateProvider.appendState(
@@ -444,9 +444,9 @@ class CollectSideOperator implements operators.IOperator {
           this.accessPattern,
           this.sideInputId,
           window,
-          this.windowCoder
+          this.windowCoder,
         ),
-        encodedElement
+        encodedElement,
       );
     }
     return operators.NonPromise;
@@ -465,10 +465,10 @@ class BufferOperator implements operators.IOperator {
   constructor(
     public transformId: string,
     transform: PTransform,
-    context: operators.OperatorContext
+    context: operators.OperatorContext,
   ) {
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
   }
 
@@ -498,7 +498,7 @@ class InMemoryStateProvider implements state.StateProvider {
 
   getState<T>(
     stateKey: fnApi.StateKey,
-    decode: (data: Uint8Array) => T
+    decode: (data: Uint8Array) => T,
   ): state.MaybePromise<T> {
     return {
       type: "value",
@@ -512,7 +512,7 @@ class InMemoryStateProvider implements state.StateProvider {
 
   getStateEntry<T>(stateKey: fnApi.StateKey) {
     const cacheKey = Buffer.from(fnApi.StateKey.toBinary(stateKey)).toString(
-      "base64"
+      "base64",
     );
     if (!this.chunks.has(cacheKey)) {
       this.chunks.set(cacheKey, []);
diff --git a/sdks/typescript/src/apache_beam/runners/flink.ts b/sdks/typescript/src/apache_beam/runners/flink.ts
index db6c7e8b0a1f..c34a50c1939c 100644
--- a/sdks/typescript/src/apache_beam/runners/flink.ts
+++ b/sdks/typescript/src/apache_beam/runners/flink.ts
@@ -39,7 +39,7 @@ export function flinkRunner(runnerOptions: Object = {}): Runner {
   return new (class extends Runner {
     async runPipeline(
       pipeline: Pipeline,
-      options: Object = {}
+      options: Object = {},
     ): Promise<PipelineResult> {
       const allOptions = {
         ...defaultOptions,
@@ -54,7 +54,7 @@ export function flinkRunner(runnerOptions: Object = {}): Runner {
       }
       if (!allOptions.artifactsDir) {
         allOptions.artifactsDir = fs.mkdtempSync(
-          path.join(os.tmpdir(), "flinkArtifactsDir")
+          path.join(os.tmpdir(), "flinkArtifactsDir"),
         );
       }
 
@@ -62,8 +62,8 @@ export function flinkRunner(runnerOptions: Object = {}): Runner {
         allOptions.flinkJobServerJar ||
         (await JavaJarService.cachedJar(
           await JavaJarService.gradleToJar(
-            `runners:flink:${allOptions.flinkVersion}:job-server:shadowJar`
-          )
+            `runners:flink:${allOptions.flinkVersion}:job-server:shadowJar`,
+          ),
         ));
       const jobServer = new JavaJarService(jobServerJar, [
         "--flink-master",
diff --git a/sdks/typescript/src/apache_beam/runners/portable_runner/runner.ts b/sdks/typescript/src/apache_beam/runners/portable_runner/runner.ts
index 9b36cc6f0ab0..ad73d4106518 100644
--- a/sdks/typescript/src/apache_beam/runners/portable_runner/runner.ts
+++ b/sdks/typescript/src/apache_beam/runners/portable_runner/runner.ts
@@ -67,7 +67,7 @@ class PortableRunnerPipelineResult extends PipelineResult {
   constructor(
     runner: PortableRunner,
     jobId: string,
-    completionCallbacks: completionCallback[]
+    completionCallbacks: completionCallback[],
   ) {
     super();
     this.runner = runner;
@@ -142,7 +142,7 @@ export class PortableRunner extends Runner {
 
   constructor(
     options: string | { jobEndpoint: string; [others: string]: any },
-    private jobService: JobService | undefined = undefined
+    private jobService: JobService | undefined = undefined,
   ) {
     super();
     if (typeof options === "string") {
@@ -161,7 +161,7 @@ export class PortableRunner extends Runner {
         new GrpcTransport({
           host: this.defaultOptions?.jobEndpoint,
           channelCredentials: ChannelCredentials.createInsecure(),
-        })
+        }),
       );
     }
     return this.client;
@@ -184,19 +184,19 @@ export class PortableRunner extends Runner {
 
   async runPipeline(
     pipeline: runnerApiProto.Pipeline,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ): Promise<PipelineResult> {
     return this.runPipelineWithProto(pipeline, options);
   }
 
   async runPipelineWithProto(
     pipeline: runnerApiProto.Pipeline,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ) {
     options = { ...this.defaultOptions, ...(options || {}) };
 
     for (const [_, pcoll] of Object.entries(
-      pipeline.components!.pcollections
+      pipeline.components!.pcollections,
     )) {
       if (pcoll.isBounded == runnerApiProto.IsBounded_Enum.UNBOUNDED) {
         (options as any).streaming = true;
@@ -221,7 +221,7 @@ export class PortableRunner extends Runner {
     // Replace the default environment according to the pipeline options.
     pipeline = runnerApiProto.Pipeline.clone(pipeline);
     for (const [envId, env] of Object.entries(
-      pipeline.components!.environments
+      pipeline.components!.environments,
     )) {
       if (env.urn === environments.TYPESCRIPT_DEFAULT_ENVIRONMENT_URN) {
         if (loopbackAddress) {
@@ -233,7 +233,7 @@ export class PortableRunner extends Runner {
             environments.asDockerEnvironment(
               env,
               (options as any)?.sdkContainerImage ||
-                DOCKER_BASE + ":" + version.replace("-SNAPSHOT", ".dev")
+                DOCKER_BASE + ":" + version.replace("-SNAPSHOT", ".dev"),
             );
           const deps = pipeline.components!.environments[envId].dependencies;
 
@@ -244,7 +244,7 @@ export class PortableRunner extends Runner {
             ["pack", "--pack-destination", tmpDir],
             {
               encoding: "latin1",
-            }
+            },
           );
           if (result.status === 0) {
             console.debug(result.stdout);
@@ -252,7 +252,7 @@ export class PortableRunner extends Runner {
             throw new Error(result.output);
           }
           const packFile = path.resolve(
-            path.join(tmpDir, result.stdout.trim())
+            path.join(tmpDir, result.stdout.trim()),
           );
           deps.push(fileArtifact(packFile, "beam:artifact:type:npm:v1"));
 
@@ -267,8 +267,8 @@ export class PortableRunner extends Runner {
                     fileArtifact(
                       path,
                       "beam:artifact:type:npm_dep:v1",
-                      new TextEncoder().encode(dep)
-                    )
+                      new TextEncoder().encode(dep),
+                    ),
                   );
                 }
               }
@@ -296,8 +296,8 @@ export class PortableRunner extends Runner {
           Object.entries(options).map(([k, v]) => [
             `beam:option:${camel_to_snake(k)}:v1`,
             v,
-          ])
-        )
+          ]),
+        ),
       );
     }
     const client = await this.getClient();
@@ -314,10 +314,10 @@ export class PortableRunner extends Runner {
           new GrpcTransport({
             host: prepareResponse.artifactStagingEndpoint.url,
             channelCredentials: ChannelCredentials.createInsecure(),
-          })
+          }),
         ),
         prepareResponse.stagingSessionToken,
-        "/"
+        "/",
       );
     }
 
@@ -340,7 +340,7 @@ export class PortableRunner extends Runner {
 function fileArtifact(
   filePath: string,
   roleUrn: string,
-  rolePayload: Uint8Array | undefined = undefined
+  rolePayload: Uint8Array | undefined = undefined,
 ) {
   const hasher = crypto.createHash("sha256");
   hasher.update(fs.readFileSync(filePath));
diff --git a/sdks/typescript/src/apache_beam/runners/runner.ts b/sdks/typescript/src/apache_beam/runners/runner.ts
index 3679cdad9ac2..b5e69cdc7256 100644
--- a/sdks/typescript/src/apache_beam/runners/runner.ts
+++ b/sdks/typescript/src/apache_beam/runners/runner.ts
@@ -38,8 +38,8 @@ export class PipelineResult {
     return Object.fromEntries(
       metrics.aggregateMetrics(
         await this.rawMetrics(),
-        "beam:metric:user:sum_int64:v1"
-      )
+        "beam:metric:user:sum_int64:v1",
+      ),
     );
   }
 
@@ -47,8 +47,8 @@ export class PipelineResult {
     return Object.fromEntries(
       metrics.aggregateMetrics(
         await this.rawMetrics(),
-        "beam:metric:user:distribution_int64:v1"
-      )
+        "beam:metric:user:distribution_int64:v1",
+      ),
     );
   }
 }
@@ -101,7 +101,7 @@ export abstract class Runner {
    */
   async run(
     pipeline: (root: Root) => PValue<any> | Promise<PValue<any>>,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ): Promise<PipelineResult> {
     const pipelineResult = await this.runAsync(pipeline, options);
     const finalState = await pipelineResult.waitUntilFinish();
@@ -119,7 +119,7 @@ export abstract class Runner {
    */
   async runAsync(
     pipeline: (root: Root) => PValue<any> | Promise<PValue<any>>,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ): Promise<PipelineResult> {
     const p = new Pipeline();
     await pipeline(new Root(p));
@@ -128,7 +128,7 @@ export abstract class Runner {
 
   abstract runPipeline(
     pipeline: runnerApi.Pipeline,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ): Promise<PipelineResult>;
 }
 
@@ -136,7 +136,7 @@ export function defaultRunner(defaultOptions: Object): Runner {
   return new (class extends Runner {
     async runPipeline(
       pipeline: runnerApi.Pipeline,
-      options: Object = {}
+      options: Object = {},
     ): Promise<PipelineResult> {
       const directRunner =
         require("./direct_runner").directRunner(defaultOptions);
diff --git a/sdks/typescript/src/apache_beam/runners/universal.ts b/sdks/typescript/src/apache_beam/runners/universal.ts
index 2a30570adf64..01fd0c88c919 100644
--- a/sdks/typescript/src/apache_beam/runners/universal.ts
+++ b/sdks/typescript/src/apache_beam/runners/universal.ts
@@ -27,14 +27,14 @@ export function universalRunner(runnerOptions: {
   return new (class extends Runner {
     async runPipeline(
       pipeline: Pipeline,
-      options: Object = {}
+      options: Object = {},
     ): Promise<PipelineResult> {
       return new PortableRunner(
         runnerOptions as any,
         PythonService.forModule(
           "apache_beam.runners.portability.local_job_service_main",
-          ["--port", "{{PORT}}"]
-        )
+          ["--port", "{{PORT}}"],
+        ),
       ).runPipeline(pipeline, { directEmbedDockerPython: true, ...options });
     }
   })();
diff --git a/sdks/typescript/src/apache_beam/testing/assert.ts b/sdks/typescript/src/apache_beam/testing/assert.ts
index 3e7fca2fb5a8..062ef229cfac 100644
--- a/sdks/typescript/src/apache_beam/testing/assert.ts
+++ b/sdks/typescript/src/apache_beam/testing/assert.ts
@@ -45,7 +45,7 @@ function callAssertDeepEqual(a, b) {
  *```
  */
 export function assertDeepEqual<T>(
-  expected: T[]
+  expected: T[],
 ): beam.PTransform<beam.PCollection<T>, void> {
   return beam.withName(
     `assertDeepEqual(${JSON.stringify(expected).substring(0, 100)})`,
@@ -54,15 +54,15 @@ export function assertDeepEqual<T>(
         assertContentsSatisfies((actual: T[]) => {
           const actualArray: T[] = [...actual];
           expected.sort((a, b) =>
-            JSON.stringify(a) < JSON.stringify(b) ? -1 : 1
+            JSON.stringify(a) < JSON.stringify(b) ? -1 : 1,
           );
           actualArray.sort((a, b) =>
-            JSON.stringify(a) < JSON.stringify(b) ? -1 : 1
+            JSON.stringify(a) < JSON.stringify(b) ? -1 : 1,
           );
           callAssertDeepEqual(actualArray, expected);
-        })
+        }),
       );
-    }
+    },
   );
 }
 
@@ -74,7 +74,7 @@ export function assertDeepEqual<T>(
  * of the provided elements is not well determined.
  */
 export function assertContentsSatisfies<T>(
-  check: (actual: T[]) => void
+  check: (actual: T[]) => void,
 ): beam.PTransform<beam.PCollection<T>, void> {
   function expand(pcoll: beam.PCollection<T>) {
     // We provide some value here to ensure there is at least one element
@@ -97,13 +97,13 @@ export function assertContentsSatisfies<T>(
             kv.value?.filter((o) => o.tag === "actual").map((o) => o.value) ||
             [];
           check(actual);
-        })
+        }),
       );
   }
 
   return beam.withName(
     `assertContentsSatisfies(${beam.extractName(check)})`,
-    expand
+    expand,
   );
 }
 
diff --git a/sdks/typescript/src/apache_beam/testing/multi_pipeline_runner.ts b/sdks/typescript/src/apache_beam/testing/multi_pipeline_runner.ts
index 4389828ef20c..e7f4fb7acca1 100644
--- a/sdks/typescript/src/apache_beam/testing/multi_pipeline_runner.ts
+++ b/sdks/typescript/src/apache_beam/testing/multi_pipeline_runner.ts
@@ -57,7 +57,7 @@ export class MultiPipelineRunner extends Runner {
 
   constructor(
     private underlying: Runner,
-    private options: PipelineOptions = {}
+    private options: PipelineOptions = {},
   ) {
     super();
   }
@@ -74,7 +74,7 @@ export class MultiPipelineRunner extends Runner {
 
   async runAsync(
     pipeline: (root: Root) => PValue<any> | Promise<PValue<any>>,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ): Promise<PipelineResult> {
     if (this.nextTestName === undefined) {
       this.setNextTestName("pipeline");
@@ -84,7 +84,7 @@ export class MultiPipelineRunner extends Runner {
     await new Root(p).applyAsync(
       withName(this.nextTestName!, async (root) => {
         await pipeline(root);
-      })
+      }),
     );
     this.nextTestName = undefined;
     return this.runPipeline(p.getProto());
@@ -92,7 +92,7 @@ export class MultiPipelineRunner extends Runner {
 
   async runPipeline(
     pipeline: runnerApi.Pipeline,
-    options?: PipelineOptions
+    options?: PipelineOptions,
   ): Promise<PipelineResult> {
     if (options) {
       throw new Error("Per-pipeline options not supported.");
@@ -108,13 +108,13 @@ export class MultiPipelineRunner extends Runner {
     console.log(this.allPipelines);
     const pipelineResult = await this.underlying.runPipeline(
       this.allPipelines,
-      this.options
+      this.options,
     );
     const finalState = await pipelineResult.waitUntilFinish();
     if (finalState != jobApi.JobState_Enum.DONE) {
       // TODO: Grab the last/most severe error message?
       throw new Error(
-        "Job finished in state " + jobApi.JobState_Enum[finalState]
+        "Job finished in state " + jobApi.JobState_Enum[finalState],
       );
     }
     this.allPipelines = undefined;
@@ -147,23 +147,23 @@ export class MultiPipelineRunner extends Runner {
     }
     mergeComponents(
       pipeline.components?.transforms,
-      this.allPipelines.components?.transforms
+      this.allPipelines.components?.transforms,
     );
     mergeComponents(
       pipeline.components?.pcollections,
-      this.allPipelines.components?.pcollections
+      this.allPipelines.components?.pcollections,
     );
     mergeComponents(
       pipeline.components?.coders,
-      this.allPipelines.components?.coders
+      this.allPipelines.components?.coders,
     );
     mergeComponents(
       pipeline.components?.windowingStrategies,
-      this.allPipelines.components?.windowingStrategies
+      this.allPipelines.components?.windowingStrategies,
     );
     mergeComponents(
       pipeline.components?.environments,
-      this.allPipelines.components?.environments
+      this.allPipelines.components?.environments,
     );
     this.allPipelines.requirements = [
       ...new Set([...this.allPipelines.requirements, ...pipeline.requirements]),
diff --git a/sdks/typescript/src/apache_beam/transforms/create.ts b/sdks/typescript/src/apache_beam/transforms/create.ts
index bd654340386a..d08704086007 100644
--- a/sdks/typescript/src/apache_beam/transforms/create.ts
+++ b/sdks/typescript/src/apache_beam/transforms/create.ts
@@ -27,7 +27,7 @@ import { Root, PCollection } from "../pvalue";
  */
 export function create<T>(
   elements: T[],
-  reshuffle: boolean = true
+  reshuffle: boolean = true,
 ): PTransform<Root, PCollection<T>> {
   function create(root: Root): PCollection<T> {
     const pcoll = root
diff --git a/sdks/typescript/src/apache_beam/transforms/external.ts b/sdks/typescript/src/apache_beam/transforms/external.ts
index 155e260f6a52..700e9cb86863 100644
--- a/sdks/typescript/src/apache_beam/transforms/external.ts
+++ b/sdks/typescript/src/apache_beam/transforms/external.ts
@@ -71,24 +71,24 @@ const defaultRawExternalTransformOptions: RawExternalTransformOptions = {
 // a cleaner way to specify them than using internal.WithCoderInternal.
 export function rawExternalTransform<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 >(
   urn: string,
   payload: Uint8Array | { [key: string]: any },
   serviceProviderOrAddress: string | (() => Promise<service.Service>),
-  options: RawExternalTransformOptions = {}
+  options: RawExternalTransformOptions = {},
 ): transform.AsyncPTransform<InputT, OutputT> {
   return new RawExternalTransform(
     urn,
     payload,
     serviceProviderOrAddress,
-    options
+    options,
   );
 }
 
 class RawExternalTransform<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > extends transform.AsyncPTransformClass<InputT, OutputT> {
   static namespaceCounter = 0;
   static freshNamespace() {
@@ -103,7 +103,7 @@ class RawExternalTransform<
     private urn: string,
     payload: Uint8Array | { [key: string]: any },
     serviceProviderOrAddress: string | (() => Promise<service.Service>),
-    options: RawExternalTransformOptions
+    options: RawExternalTransformOptions,
   ) {
     super("External(" + urn + ")");
     this.options = { ...defaultRawExternalTransformOptions, ...options };
@@ -126,7 +126,7 @@ class RawExternalTransform<
   async expandInternalAsync(
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ): Promise<OutputT> {
     const pipelineComponents = pipeline.getProto().components!;
     const namespace = RawExternalTransform.freshNamespace();
@@ -155,7 +155,7 @@ class RawExternalTransform<
     }
 
     for (const [output, coder] of Object.entries(
-      this.options.requestedOutputCoders!
+      this.options.requestedOutputCoders!,
     )) {
       request.outputCoderRequests[output] = pipeline.getCoderId(coder);
     }
@@ -164,11 +164,11 @@ class RawExternalTransform<
     Object.assign(request.components!.coders, pipelineComponents.coders);
     Object.assign(
       request.components!.windowingStrategies,
-      pipelineComponents.windowingStrategies
+      pipelineComponents.windowingStrategies,
     );
     Object.assign(
       request.components!.environments,
-      pipelineComponents.environments
+      pipelineComponents.environments,
     );
 
     const service = await this.serviceProvider();
@@ -178,7 +178,7 @@ class RawExternalTransform<
       new GrpcTransport({
         host: address,
         channelCredentials: ChannelCredentials.createInsecure(),
-      })
+      }),
     );
 
     try {
@@ -190,7 +190,7 @@ class RawExternalTransform<
 
       response.components = await this.resolveArtifacts(
         response.components!,
-        address
+        address,
       );
 
       return this.splice(pipeline, transformProto, response, namespace);
@@ -208,12 +208,12 @@ class RawExternalTransform<
    */
   async resolveArtifacts(
     components: runnerApi.Components,
-    address: string
+    address: string,
   ): Promise<runnerApi.Components> {
     // Don't even bother creating a connection if there are no dependencies.
     if (
       Object.values(components.environments).every(
-        (env) => env.dependencies.length === 0
+        (env) => env.dependencies.length === 0,
       )
     ) {
       return components;
@@ -225,7 +225,7 @@ class RawExternalTransform<
       new GrpcTransport({
         host: address,
         channelCredentials: ChannelCredentials.createInsecure(),
-      })
+      }),
     );
 
     // For each new environment, convert (if needed) all dependencies into
@@ -237,12 +237,12 @@ class RawExternalTransform<
             let result: runnerApi.ArtifactInformation[] = [];
             for await (const dep of artifacts.resolveArtifacts(
               artifactClient,
-              env.dependencies
+              env.dependencies,
             )) {
               result.push(dep);
             }
             return result;
-          })()
+          })(),
         );
       }
     }
@@ -254,11 +254,11 @@ class RawExternalTransform<
     pipeline: Pipeline,
     transformProto: runnerApi.PTransform,
     response: ExpansionResponse,
-    namespace: string
+    namespace: string,
   ): OutputT {
     function copyNamespaceComponents<T>(
       src: { [key: string]: T },
-      dest: { [key: string]: T }
+      dest: { [key: string]: T },
     ) {
       for (const [id, proto] of Object.entries(src)) {
         if (id.startsWith(namespace)) {
@@ -274,14 +274,14 @@ class RawExternalTransform<
     // Some SDKs enforce input naming conventions.
     const newTags = difference(
       new Set(Object.keys(response.transform!.inputs)),
-      new Set(Object.keys(transformProto.inputs))
+      new Set(Object.keys(transformProto.inputs)),
     );
     if (newTags.length > 1) {
       throw new Error("Ambiguous renaming of tags.");
     } else if (newTags.length === 1) {
       const missingTags = difference(
         new Set(Object.keys(transformProto.inputs)),
-        new Set(Object.keys(response.transform!.inputs))
+        new Set(Object.keys(response.transform!.inputs)),
       );
       transformProto.inputs[newTags[0]] = transformProto.inputs[missingTags[0]];
       delete transformProto.inputs[missingTags[0]];
@@ -292,13 +292,13 @@ class RawExternalTransform<
       Object.keys(response.transform!.inputs).map((k) => [
         response.transform!.inputs[k],
         transformProto.inputs[k],
-      ])
+      ]),
     );
     response.transform!.inputs = Object.fromEntries(
       Object.entries(response.transform!.inputs).map(([k, v]) => [
         k,
         renamedInputs[v],
-      ])
+      ]),
     );
     for (const t of Object.values(response.components!.transforms)) {
       t.inputs = Object.fromEntries(
@@ -307,7 +307,7 @@ class RawExternalTransform<
           renamedInputs[v] !== null && renamedInputs[v] !== undefined
             ? renamedInputs[v]
             : v,
-        ])
+        ]),
       );
     }
 
@@ -320,23 +320,23 @@ class RawExternalTransform<
     pipeline.getProto().requirements.push(...response.requirements);
     copyNamespaceComponents(
       response.components!.transforms,
-      pipelineComponents!.transforms
+      pipelineComponents!.transforms,
     );
     copyNamespaceComponents(
       response.components!.pcollections,
-      pipelineComponents!.pcollections
+      pipelineComponents!.pcollections,
     );
     copyNamespaceComponents(
       response.components!.coders,
-      pipelineComponents!.coders
+      pipelineComponents!.coders,
     );
     copyNamespaceComponents(
       response.components!.environments,
-      pipelineComponents!.environments
+      pipelineComponents!.environments,
     );
     copyNamespaceComponents(
       response.components!.windowingStrategies,
-      pipelineComponents!.windowingStrategies
+      pipelineComponents!.windowingStrategies,
     );
 
     // Ensure we understand the resulting coders.
@@ -360,7 +360,7 @@ class RawExternalTransform<
       } else if (outputKeys.length === 1) {
         return new PCollection(
           pipeline,
-          response.transform!.outputs[outputKeys[0]]
+          response.transform!.outputs[outputKeys[0]],
         ) as OutputT;
       }
     }
@@ -368,14 +368,14 @@ class RawExternalTransform<
       Object.entries(response.transform!.outputs).map(([k, v]) => [
         k,
         new PCollection(pipeline, v),
-      ])
+      ]),
     ) as OutputT;
   }
 }
 
 function encodeSchemaPayload(
   payload: any,
-  schema: Schema | undefined = undefined
+  schema: Schema | undefined = undefined,
 ): Uint8Array {
   const encoded = new Writer();
   if (!schema) {
diff --git a/sdks/typescript/src/apache_beam/transforms/flatten.ts b/sdks/typescript/src/apache_beam/transforms/flatten.ts
index 99fcd74e8a05..712cf85c2578 100644
--- a/sdks/typescript/src/apache_beam/transforms/flatten.ts
+++ b/sdks/typescript/src/apache_beam/transforms/flatten.ts
@@ -32,7 +32,7 @@ export function flatten<T>(): PTransform<PCollection<T>[], PCollection<T>> {
   function expandInternal(
     inputs: PCollection<T>[],
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     transformProto.spec = runnerApi.FunctionSpec.create({
       urn: flatten.urn,
@@ -40,7 +40,7 @@ export function flatten<T>(): PTransform<PCollection<T>[], PCollection<T>> {
 
     // TODO: UnionCoder if they're not the same?
     const coders = new Set(
-      inputs.map((pc) => pipeline.context.getPCollectionCoderId(pc))
+      inputs.map((pc) => pipeline.context.getPCollectionCoderId(pc)),
     );
     const coder =
       coders.size === 1 ? [...coders][0] : new GeneralObjectCoder<T>();
diff --git a/sdks/typescript/src/apache_beam/transforms/group_and_combine.ts b/sdks/typescript/src/apache_beam/transforms/group_and_combine.ts
index 777d89636645..2cf2d44281af 100644
--- a/sdks/typescript/src/apache_beam/transforms/group_and_combine.ts
+++ b/sdks/typescript/src/apache_beam/transforms/group_and_combine.ts
@@ -80,7 +80,7 @@ export class GroupBy<T, K> extends PTransformClass<
    */
   constructor(
     key: string | string[] | ((element: T) => K),
-    keyName: string | undefined = undefined
+    keyName: string | undefined = undefined,
   ) {
     super();
     [this.keyFn, this.keyNames] = extractFnAndName(key, keyName || "key");
@@ -99,15 +99,15 @@ export class GroupBy<T, K> extends PTransformClass<
   combining<I>(
     expr: string | ((element: T) => I),
     combiner: Combiner<I>,
-    resultName: string
+    resultName: string,
   ) {
     return withName(
       extractName(this),
       new GroupByAndCombine(this.keyFn, this.keyNames, []).combining(
         expr,
         combiner,
-        resultName
-      )
+        resultName,
+      ),
     );
   }
 }
@@ -122,11 +122,11 @@ export class GroupBy<T, K> extends PTransformClass<
  */
 export function groupBy<T, K>(
   key: string | string[] | ((element: T) => K),
-  keyName: string | undefined = undefined
+  keyName: string | undefined = undefined,
 ): GroupBy<T, K> {
   return withName(
     `groupBy(${extractName(key)}`,
-    new GroupBy<T, K>(key, keyName)
+    new GroupBy<T, K>(key, keyName),
   );
 }
 
@@ -153,15 +153,15 @@ export class GroupGlobally<T> extends PTransformClass<
   combining<I>(
     expr: string | ((element: T) => I),
     combiner: Combiner<I>,
-    resultName: string
+    resultName: string,
   ) {
     return withName(
       extractName(this),
       new GroupByAndCombine((_) => null, undefined, []).combining(
         expr,
         combiner,
-        resultName
-      )
+        resultName,
+      ),
     );
   }
 }
@@ -190,7 +190,7 @@ class GroupByAndCombine<T, O> extends PTransformClass<
   constructor(
     keyFn: (element: T) => any,
     keyNames: string | string[] | undefined,
-    combiners: CombineSpec<T, any, any>[]
+    combiners: CombineSpec<T, any, any>[],
   ) {
     super();
     this.keyFn = keyFn;
@@ -202,7 +202,7 @@ class GroupByAndCombine<T, O> extends PTransformClass<
   combining<I, O>(
     expr: string | ((element: T) => I),
     combiner: Combiner<I>,
-    resultName: string // TODO: (Unique names) Optionally derive from expr and combineFn?
+    resultName: string, // TODO: (Unique names) Optionally derive from expr and combineFn?
   ) {
     return withName(
       extractName(this),
@@ -215,8 +215,8 @@ class GroupByAndCombine<T, O> extends PTransformClass<
             combineFn: toCombineFn(combiner),
             resultName: resultName,
           },
-        ])
-      )
+        ]),
+      ),
     );
   }
 
@@ -231,8 +231,8 @@ class GroupByAndCombine<T, O> extends PTransformClass<
       })
       .apply(
         internal.combinePerKey(
-          multiCombineFn(this_.combiners.map((c) => c.combineFn))
-        )
+          multiCombineFn(this_.combiners.map((c) => c.combineFn)),
+        ),
       )
       .map(function constructResult(kv) {
         const result = {};
@@ -259,7 +259,7 @@ export function countPerElement<T>(): PTransform<
 > {
   return withName(
     "countPerElement",
-    groupBy((e) => e, "element").combining((e) => e, count, "count")
+    groupBy((e) => e, "element").combining((e) => e, count, "count"),
   );
 }
 
@@ -270,7 +270,7 @@ export function countGlobally<T>(): PTransform<
   return withName("countGlobally", (input) =>
     input
       .apply(new GroupGlobally().combining((e) => e, count, "count"))
-      .map((o) => o.count)
+      .map((o) => o.count),
   );
 }
 
@@ -293,7 +293,7 @@ interface CombineSpec<T, I, O> {
  * commutative and associative).
  */
 export function binaryCombineFn<I>(
-  combiner: (a: I, b: I) => I
+  combiner: (a: I, b: I) => I,
 ): CombineFn<I, I | undefined, I> {
   return {
     createAccumulator: () => undefined,
@@ -307,7 +307,7 @@ export function binaryCombineFn<I>(
 // TODO: (Typescript) Is there a way to indicate type parameters match the above?
 function multiCombineFn(
   combineFns: CombineFn<any, any, any>[],
-  batchSize: number = 100
+  batchSize: number = 100,
 ): CombineFn<any[], any[], any[]> {
   return {
     createAccumulator: () => combineFns.map((fn) => fn.createAccumulator()),
@@ -353,7 +353,7 @@ function multiCombineFn(
 // TODO: Consider adding valueFn(s) rather than using the full value.
 export function coGroupBy<T, K>(
   key: string | string[] | ((element: T) => K),
-  keyName: string | undefined = undefined
+  keyName: string | undefined = undefined,
 ): PTransform<
   { [key: string]: PCollection<any> },
   PCollection<{ key: K; values: { [key: string]: Iterable<any> } }>
@@ -370,22 +370,22 @@ export function coGroupBy<T, K>(
             key: keyFn(element),
             tag,
             element,
-          }))
-        )
+          })),
+        ),
       );
       return P(tagged)
         .apply(flatten())
         .apply(groupBy("key"))
         .map(function groupValues({ key, value }) {
           const groupedValues: { [key: string]: any[] } = Object.fromEntries(
-            tags.map((tag) => [tag, []])
+            tags.map((tag) => [tag, []]),
           );
           for (const { tag, element } of value) {
             groupedValues[tag].push(element);
           }
           return { key, values: groupedValues };
         });
-    }
+    },
   );
 }
 
@@ -398,7 +398,7 @@ export function coGroupBy<T, K>(
 // ): [(element: T) => K, P | P[]] {
 function extractFnAndName<T, K>(
   extractor: string | string[] | ((T) => K),
-  defaultName: string
+  defaultName: string,
 ): [(T) => K, string | string[]] {
   if (
     typeof extractor === "string" ||
diff --git a/sdks/typescript/src/apache_beam/transforms/internal.ts b/sdks/typescript/src/apache_beam/transforms/internal.ts
index 9097a4222ba7..e39785d8d598 100644
--- a/sdks/typescript/src/apache_beam/transforms/internal.ts
+++ b/sdks/typescript/src/apache_beam/transforms/internal.ts
@@ -49,7 +49,7 @@ export function impulse(): PTransform<Root, PCollection<Uint8Array>> {
   function expandInternal(
     input: Root,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     transformProto.spec = runnerApi.FunctionSpec.create({
       urn: impulse.urn,
@@ -66,14 +66,14 @@ impulse.urn = "beam:transform:impulse:v1";
 
 // TODO: (API) Should we offer a method on PCollection to do this?
 export function withCoderInternal<T>(
-  coder: Coder<T>
+  coder: Coder<T>,
 ): PTransform<PCollection<T>, PCollection<T>> {
   return withName(
     `withCoderInternal(${extractName(coder)})`,
     (
       input: PCollection<T>,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
+      transformProto: runnerApi.PTransform,
     ) => {
       // IDENTITY rather than Flatten for better fusion.
       transformProto.spec = {
@@ -84,12 +84,12 @@ export function withCoderInternal<T>(
               urn: urns.IDENTITY_DOFN_URN,
               payload: undefined!,
             }),
-          })
+          }),
         ),
       };
 
       return pipeline.createPCollectionInternal<T>(coder);
-    }
+    },
   );
 }
 
@@ -108,7 +108,7 @@ export function withCoderInternal<T>(
  * invoke cross-language transforms.
  */
 export function withRowCoder<T extends Object>(
-  exemplar: T
+  exemplar: T,
 ): PTransform<PCollection<T>, PCollection<T>> {
   return withCoderInternal(RowCoder.fromJSON(exemplar));
 }
@@ -131,7 +131,7 @@ export function groupByKey<K, V>(): PTransform<
   function expandInternal(
     input: PCollection<KV<K, V>>,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     const pipelineComponents: runnerApi.Components =
       pipeline.getProto().components!;
@@ -144,8 +144,8 @@ export function groupByKey<K, V>(): PTransform<
       return input
         .apply(
           withCoderInternal(
-            new KVCoder(new GeneralObjectCoder(), new GeneralObjectCoder())
-          )
+            new KVCoder(new GeneralObjectCoder(), new GeneralObjectCoder()),
+          ),
         )
         .apply(groupByKey());
     }
@@ -184,12 +184,12 @@ groupByKey.urn = "beam:transform:group_by_key:v1";
  * help reduce the original data into a single aggregator per key per worker.
  */
 export function combinePerKey<K, InputT, AccT, OutputT>(
-  combineFn: CombineFn<InputT, AccT, OutputT>
+  combineFn: CombineFn<InputT, AccT, OutputT>,
 ): PTransform<PCollection<KV<K, InputT>>, PCollection<KV<K, OutputT>>> {
   function expandInternal(
     input: PCollection<KV<any, InputT>>,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     const pipelineComponents: runnerApi.Components =
       pipeline.getProto().components!;
@@ -198,7 +198,7 @@ export function combinePerKey<K, InputT, AccT, OutputT>(
     try {
       // If this fails, we cannot lift, so we skip setting the liftable URN.
       CombinePerKeyPrecombineOperator.checkSupportsWindowing(
-        pipelineComponents.windowingStrategies[inputProto.windowingStrategyId]
+        pipelineComponents.windowingStrategies[inputProto.windowingStrategyId],
       );
 
       // Ensure the input is using the KV coder.
@@ -207,14 +207,14 @@ export function combinePerKey<K, InputT, AccT, OutputT>(
         return input
           .apply(
             withCoderInternal(
-              new KVCoder(new GeneralObjectCoder(), new GeneralObjectCoder())
-            )
+              new KVCoder(new GeneralObjectCoder(), new GeneralObjectCoder()),
+            ),
           )
           .apply(combinePerKey(combineFn));
       }
 
       const inputValueCoder = pipeline.context.getCoder<InputT>(
-        inputCoderProto.componentCoderIds[1]
+        inputCoderProto.componentCoderIds[1],
       );
 
       transformProto.spec = runnerApi.FunctionSpec.create({
@@ -227,7 +227,7 @@ export function combinePerKey<K, InputT, AccT, OutputT>(
           accumulatorCoderId: pipeline.context.getCoderId(
             combineFn.accumulatorCoder
               ? combineFn.accumulatorCoder(inputValueCoder)
-              : new GeneralObjectCoder()
+              : new GeneralObjectCoder(),
           ),
         }),
       });
@@ -250,16 +250,16 @@ export function combinePerKey<K, InputT, AccT, OutputT>(
           for (const value of kv.value) {
             accumulators[ix % 3] = combineFn.addInput(
               accumulators[ix % 3],
-              value
+              value,
             );
           }
           return {
             key: kv.key,
             value: combineFn.extractOutput(
-              combineFn.mergeAccumulators(accumulators)
+              combineFn.mergeAccumulators(accumulators),
             ),
           };
-        })
+        }),
       );
   }
 
diff --git a/sdks/typescript/src/apache_beam/transforms/pardo.ts b/sdks/typescript/src/apache_beam/transforms/pardo.ts
index 8abe24b9de9a..60a3d11aa18d 100644
--- a/sdks/typescript/src/apache_beam/transforms/pardo.ts
+++ b/sdks/typescript/src/apache_beam/transforms/pardo.ts
@@ -88,10 +88,10 @@ export interface DoFn<InputT, OutputT, ContextT = undefined> {
 export function parDo<
   InputT,
   OutputT,
-  ContextT extends Object | undefined = undefined
+  ContextT extends Object | undefined = undefined,
 >(
   doFn: DoFn<InputT, OutputT, ContextT>,
-  context: ContextT = undefined!
+  context: ContextT = undefined!,
 ): PTransform<PCollection<InputT>, PCollection<OutputT>> {
   if (extractContext(doFn)) {
     context = { ...extractContext(doFn), ...context };
@@ -99,7 +99,7 @@ export function parDo<
   function expandInternal(
     input: PCollection<InputT>,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     // Extract and populate side inputs from the context.
     const sideInputs = {};
@@ -135,8 +135,8 @@ export function parDo<
               urn: isGlobalSide
                 ? urns.GLOBAL_WINDOW_MAPPING_FN_URN
                 : mainWindowingStrategyId === sideWindowingStrategyId
-                ? urns.IDENTITY_WINDOW_MAPPING_FN_URN
-                : urns.ASSIGN_MAX_TIMESTAMP_WINDOW_MAPPING_FN_URN,
+                  ? urns.IDENTITY_WINDOW_MAPPING_FN_URN
+                  : urns.ASSIGN_MAX_TIMESTAMP_WINDOW_MAPPING_FN_URN,
               value: new Uint8Array(),
             },
           };
@@ -161,7 +161,7 @@ export function parDo<
             }),
           }),
           sideInputs: sideInputs,
-        })
+        }),
       ),
     });
 
@@ -169,7 +169,7 @@ export function parDo<
     // coder to encode the various types that exist in JS.
     // TODO: (Types) Should there be a way to specify, or better yet infer, the coder to use?
     return pipeline.createPCollectionInternal<OutputT>(
-      new GeneralObjectCoder()
+      new GeneralObjectCoder(),
     );
   }
 
@@ -198,12 +198,12 @@ export type SplitOptions = {
 // TODO: Naming.
 export function split<X extends { [key: string]: unknown }>(
   tags: string[],
-  options: SplitOptions = {}
+  options: SplitOptions = {},
 ): PTransform<PCollection<X>, { [P in keyof X]: PCollection<X[P]> }> {
   function expandInternal(
     input: PCollection<X>,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ) {
     if (options.exclusive === undefined) {
       options.exclusive = true;
@@ -229,7 +229,7 @@ export function split<X extends { [key: string]: unknown }>(
             urn: urns.SPLITTING_JS_DOFN_URN,
             payload: serializeFn(options),
           }),
-        })
+        }),
       ),
     });
 
@@ -237,9 +237,9 @@ export function split<X extends { [key: string]: unknown }>(
       tags.map((tag) => [
         tag,
         pipeline.createPCollectionInternal<X[typeof tag]>(
-          pipeline.context.getPCollectionCoderId(input)
+          pipeline.context.getPCollectionCoderId(input),
         ),
-      ])
+      ]),
     ) as { [P in keyof X]: PCollection<X[P]> };
   }
 
@@ -248,11 +248,11 @@ export function split<X extends { [key: string]: unknown }>(
 
 export function partition<T>(
   partitionFn: (element: T, numPartitions: number) => number,
-  numPartitions: number
+  numPartitions: number,
 ): PTransform<PCollection<T>, PCollection<T>[]> {
   return function partition(input: PCollection<T>) {
     const indices = Array.from({ length: numPartitions }, (v, i) =>
-      i.toString()
+      i.toString(),
     );
     const splits = input
       .map((x) => {
@@ -275,7 +275,7 @@ export function withContext<
   ContextT,
   T extends
     | DoFn<unknown, unknown, ContextT>
-    | ((input: unknown, context: ContextT) => unknown)
+    | ((input: unknown, context: ContextT) => unknown),
 >(fn: T, contextSpec: ContextT): T {
   const untypedFn = fn as any;
   untypedFn.beamPardoContextSpec = {
@@ -375,7 +375,7 @@ interface SideInputAccessor<PCollT, AccessorT, ValueT> {
 export class SideInputParam<
   PCollT,
   AccessorT,
-  ValueT
+  ValueT,
 > extends ParDoLookupParam<ValueT> {
   // Populated by user.
   pcoll: PCollection<PCollT>;
@@ -384,7 +384,7 @@ export class SideInputParam<
 
   constructor(
     pcoll: PCollection<PCollT>,
-    accessor: SideInputAccessor<PCollT, AccessorT, ValueT>
+    accessor: SideInputAccessor<PCollT, AccessorT, ValueT>,
   ) {
     super("sideInput");
     this.pcoll = pcoll;
@@ -398,7 +398,7 @@ export class SideInputParam<
 
 function copySideInputWithId<PCollT, AccessorT, ValueT>(
   sideInput: SideInputParam<PCollT, AccessorT, ValueT>,
-  id: string
+  id: string,
 ): SideInputParam<PCollT, AccessorT, ValueT> {
   const copy = Object.create(sideInput);
   copy.sideInputId = id;
@@ -407,7 +407,7 @@ function copySideInputWithId<PCollT, AccessorT, ValueT>(
 }
 
 export function iterableSideInput<T>(
-  pcoll: PCollection<T>
+  pcoll: PCollection<T>,
 ): SideInputParam<T, Iterable<T>, Iterable<T>> {
   return new SideInputParam<T, Iterable<T>, Iterable<T>>(pcoll, {
     accessPattern: "beam:side_input:iterable:v1",
@@ -417,7 +417,7 @@ export function iterableSideInput<T>(
 
 export function singletonSideInput<T>(
   pcoll: PCollection<T>,
-  defaultValue: T | undefined = undefined
+  defaultValue: T | undefined = undefined,
 ): SideInputParam<T, Iterable<T>, T> {
   return new SideInputParam<T, Iterable<T>, T>(pcoll, {
     accessPattern: "beam:side_input:iterable:v1",
@@ -444,7 +444,10 @@ export function singletonSideInput<T>(
  * The superclass of all metric accessors, such as counters and distributions.
  */
 export class Metric<T> extends ParDoUpdateParam<T> {
-  constructor(readonly metricType: string, readonly name: string) {
+  constructor(
+    readonly metricType: string,
+    readonly name: string,
+  ) {
     super("metric");
   }
 }
diff --git a/sdks/typescript/src/apache_beam/transforms/python.ts b/sdks/typescript/src/apache_beam/transforms/python.ts
index 2292f9ce0e66..d1ff02ef4c1a 100644
--- a/sdks/typescript/src/apache_beam/transforms/python.ts
+++ b/sdks/typescript/src/apache_beam/transforms/python.ts
@@ -43,12 +43,12 @@ import * as row_coder from "../coders/row_coder";
  */
 export function pythonTransform<
   InputT extends beam.PValue<any>,
-  OutputT extends beam.PValue<any>
+  OutputT extends beam.PValue<any>,
 >(
   constructor: string,
   args_or_kwargs: any[] | { [key: string]: any } | undefined = undefined,
   kwargs: { [key: string]: any } | undefined = undefined,
-  options: external.RawExternalTransformOptions = {}
+  options: external.RawExternalTransformOptions = {},
 ): beam.AsyncPTransform<InputT, OutputT> {
   let args;
   if (args_or_kwargs === undefined) {
@@ -78,9 +78,9 @@ export function pythonTransform<
     async () =>
       PythonService.forModule(
         "apache_beam.runners.portability.expansion_service_main",
-        ["--fully_qualified_name_glob=*", "--port", "{{PORT}}"]
+        ["--fully_qualified_name_glob=*", "--port", "{{PORT}}"],
       ),
-    options
+    options,
   );
 }
 
diff --git a/sdks/typescript/src/apache_beam/transforms/sql.ts b/sdks/typescript/src/apache_beam/transforms/sql.ts
index 9f0e117f2f81..f219d40b0c02 100644
--- a/sdks/typescript/src/apache_beam/transforms/sql.ts
+++ b/sdks/typescript/src/apache_beam/transforms/sql.ts
@@ -41,10 +41,10 @@ import { serviceProviderFromJavaGradleTarget } from "../utils/service";
  *    ));
  */
 export function sqlTransform<
-  InputT extends PCollection<any> | { [key: string]: PCollection<any> }
+  InputT extends PCollection<any> | { [key: string]: PCollection<any> },
 >(
   query: string,
-  inputTypes = null
+  inputTypes = null,
 ): transform.AsyncPTransform<InputT, PCollection<any>> {
   // TOOD: (API) (Typescript): How to infer input_types, or at least make it optional.
   async function expandInternal(input: InputT): Promise<PCollection<any>> {
@@ -60,7 +60,7 @@ export function sqlTransform<
             "SqlTransform can only be applied to schema'd transforms. " +
               "Please ensure the input PCollection(s) have a RowCoder, " +
               "or pass a prototypical element in as the second argument " +
-              "of SqlTransform so that one can be inferred."
+              "of SqlTransform so that one can be inferred.",
           );
         }
         return pcoll;
@@ -78,9 +78,9 @@ export function sqlTransform<
             input[tag],
             inputTypes === null || inputTypes === undefined
               ? null
-              : inputTypes[tag]
+              : inputTypes[tag],
           ),
-        ])
+        ]),
       ) as InputT;
     }
 
@@ -89,9 +89,9 @@ export function sqlTransform<
         "beam:external:java:sql:v1",
         { query: query },
         serviceProviderFromJavaGradleTarget(
-          "sdks:java:extensions:sql:expansion-service:shadowJar"
-        )
-      )
+          "sdks:java:extensions:sql:expansion-service:shadowJar",
+        ),
+      ),
     );
   }
 
diff --git a/sdks/typescript/src/apache_beam/transforms/transform.ts b/sdks/typescript/src/apache_beam/transforms/transform.ts
index 799413e02d12..0d628818bac0 100644
--- a/sdks/typescript/src/apache_beam/transforms/transform.ts
+++ b/sdks/typescript/src/apache_beam/transforms/transform.ts
@@ -84,7 +84,7 @@ export function extractName<T>(withName: T): string {
 /** @internal */
 export class AsyncPTransformClass<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > {
   beamName: string | (() => string);
 
@@ -99,7 +99,7 @@ export class AsyncPTransformClass<
   async expandInternalAsync(
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ): Promise<OutputT> {
     return this.expandAsync(input);
   }
@@ -108,7 +108,7 @@ export class AsyncPTransformClass<
 /** @internal */
 export class PTransformClass<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > extends AsyncPTransformClass<InputT, OutputT> {
   expand(input: InputT): OutputT {
     throw new Error("Method expand has not been implemented.");
@@ -121,7 +121,7 @@ export class PTransformClass<
   expandInternal(
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ): OutputT {
     return this.expand(input);
   }
@@ -129,7 +129,7 @@ export class PTransformClass<
   async expandInternalAsync(
     input: InputT,
     pipeline: Pipeline,
-    transformProto: runnerApi.PTransform
+    transformProto: runnerApi.PTransform,
   ): Promise<OutputT> {
     return this.expandInternal(input, pipeline, transformProto);
   }
@@ -143,14 +143,14 @@ export class PTransformClass<
  */
 export type AsyncPTransform<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > =
   | AsyncPTransformClass<InputT, OutputT>
   | ((input: InputT) => Promise<OutputT>)
   | ((
       input: InputT,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
+      transformProto: runnerApi.PTransform,
     ) => Promise<OutputT>);
 
 /**
@@ -179,12 +179,12 @@ export type AsyncPTransform<
  */
 export type PTransform<
   InputT extends PValue<any>,
-  OutputT extends PValue<any>
+  OutputT extends PValue<any>,
 > =
   | PTransformClass<InputT, OutputT>
   | ((input: InputT) => OutputT)
   | ((
       input: InputT,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
+      transformProto: runnerApi.PTransform,
     ) => OutputT);
diff --git a/sdks/typescript/src/apache_beam/transforms/utils.ts b/sdks/typescript/src/apache_beam/transforms/utils.ts
index 8edcbb40e923..7523ade3c8ee 100644
--- a/sdks/typescript/src/apache_beam/transforms/utils.ts
+++ b/sdks/typescript/src/apache_beam/transforms/utils.ts
@@ -30,8 +30,8 @@ export function reshuffle<T>(): PTransform<PCollection<T>, PCollection<T>> {
       .apply(
         withName(
           "groupByRandomKey",
-          groupBy((x) => Math.random())
-        )
+          groupBy((x) => Math.random()),
+        ),
       )
       .flatMap(withName("dropKeys", (kvs) => kvs.value));
   }
diff --git a/sdks/typescript/src/apache_beam/transforms/window.ts b/sdks/typescript/src/apache_beam/transforms/window.ts
index f53cc9ce97f8..ba52466d136c 100644
--- a/sdks/typescript/src/apache_beam/transforms/window.ts
+++ b/sdks/typescript/src/apache_beam/transforms/window.ts
@@ -40,7 +40,7 @@ export interface WindowFn<W extends Window> {
 export function createWindowingStrategyProto(
   pipeline: Pipeline,
   windowFn: WindowFn<any>,
-  windowingStrategyBase: runnerApi.WindowingStrategy | undefined = undefined
+  windowingStrategyBase: runnerApi.WindowingStrategy | undefined = undefined,
 ): runnerApi.WindowingStrategy {
   let result: runnerApi.WindowingStrategy;
   if (windowingStrategyBase === null || windowingStrategyBase === undefined) {
@@ -78,14 +78,14 @@ export function createWindowingStrategyProto(
  */
 export function windowInto<T, W extends Window>(
   windowFn: WindowFn<W>,
-  windowingStrategyBase: runnerApi.WindowingStrategy | undefined = undefined
+  windowingStrategyBase: runnerApi.WindowingStrategy | undefined = undefined,
 ): PTransform<PCollection<T>, PCollection<T>> {
   return withName(
     `WindowInto(${extractName(windowFn)}, ${windowingStrategyBase})`,
     (
       input: PCollection<T>,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
+      transformProto: runnerApi.PTransform,
     ) => {
       transformProto.spec = runnerApi.FunctionSpec.create({
         urn: parDo.urn,
@@ -95,29 +95,29 @@ export function windowInto<T, W extends Window>(
               urn: urns.JS_WINDOW_INTO_DOFN_URN,
               payload: serializeFn({ windowFn: windowFn }),
             }),
-          })
+          }),
         ),
       });
 
       const inputCoder = pipeline.context.getPCollectionCoderId(input);
       return pipeline.createPCollectionInternal<T>(
         inputCoder,
-        createWindowingStrategyProto(pipeline, windowFn, windowingStrategyBase)
+        createWindowingStrategyProto(pipeline, windowFn, windowingStrategyBase),
       );
-    }
+    },
   );
 }
 
 // TODO: (Cleanup) Add restrictions on moving backwards?
 export function assignTimestamps<T>(
-  timestampFn: (T, Instant) => typeof Instant
+  timestampFn: (T, Instant) => typeof Instant,
 ): PTransform<PCollection<T>, PCollection<T>> {
   return withName(
     `assignTimestamp(${extractName(timestampFn)})`,
     (
       input: PCollection<T>,
       pipeline: Pipeline,
-      transformProto: runnerApi.PTransform
+      transformProto: runnerApi.PTransform,
     ) => {
       transformProto.spec = runnerApi.FunctionSpec.create({
         urn: parDo.urn,
@@ -127,13 +127,13 @@ export function assignTimestamps<T>(
               urn: urns.JS_ASSIGN_TIMESTAMPS_DOFN_URN,
               payload: serializeFn({ func: timestampFn }),
             }),
-          })
+          }),
         ),
       });
 
       return pipeline.createPCollectionInternal<T>(
-        pipeline.context.getPCollectionCoderId(input)
+        pipeline.context.getPCollectionCoderId(input),
       );
-    }
+    },
   );
 }
diff --git a/sdks/typescript/src/apache_beam/transforms/windowings.ts b/sdks/typescript/src/apache_beam/transforms/windowings.ts
index 782ae820d855..158d4fe02f88 100644
--- a/sdks/typescript/src/apache_beam/transforms/windowings.ts
+++ b/sdks/typescript/src/apache_beam/transforms/windowings.ts
@@ -47,7 +47,7 @@ export function globalWindows(): WindowFn<GlobalWindow> {
 
 export function fixedWindows(
   sizeSeconds: number | Long,
-  offsetSeconds: Instant = Long.fromValue(0)
+  offsetSeconds: Instant = Long.fromValue(0),
 ): WindowFn<IntervalWindow> {
   // TODO: (Cleanup) Use a time library?
   const sizeMillis = secsToMillisLong(sizeSeconds);
@@ -79,7 +79,7 @@ export function fixedWindows(
 export function slidingWindows(
   sizeSeconds: number | Long,
   periodSeconds: number | Long,
-  offsetSeconds: Instant = Long.fromValue(0)
+  offsetSeconds: Instant = Long.fromValue(0),
 ): WindowFn<IntervalWindow> {
   const sizeMillis = secsToMillisLong(sizeSeconds);
   const periodMillis = secsToMillisLong(periodSeconds);
@@ -150,13 +150,13 @@ requireForSerialization("apache-beam/transforms/windowings", exports);
 requireForSerialization("apache-beam/transforms/windowings", { millisToProto });
 requireForSerialization(
   "apache-beam/transforms/windowings",
-  FixedWindowsPayload
+  FixedWindowsPayload,
 );
 requireForSerialization(
   "apache-beam/transforms/windowings",
-  SlidingWindowsPayload
+  SlidingWindowsPayload,
 );
 requireForSerialization(
   "apache-beam/transforms/windowings",
-  SessionWindowsPayload
+  SessionWindowsPayload,
 );
diff --git a/sdks/typescript/src/apache_beam/utils/service.ts b/sdks/typescript/src/apache_beam/utils/service.ts
index b53dbd626fb8..4b73e7ae53fc 100644
--- a/sdks/typescript/src/apache_beam/utils/service.ts
+++ b/sdks/typescript/src/apache_beam/utils/service.ts
@@ -71,7 +71,7 @@ class SubprocessServiceCache {
       [...this.services.values()].map((service) => {
         service.cached = false;
         return service.stop();
-      })
+      }),
     );
   }
 }
@@ -89,7 +89,7 @@ export class SubprocessService {
   constructor(
     cmd: string,
     args: string[],
-    name: string | undefined = undefined
+    name: string | undefined = undefined,
   ) {
     this.cmd = cmd;
     this.args = args;
@@ -108,7 +108,7 @@ export class SubprocessService {
 
   static createCache(): SubprocessServiceCache {
     SubprocessService.cache = new SubprocessServiceCache(
-      SubprocessService.cache
+      SubprocessService.cache,
     );
     return this.cache!;
   }
@@ -129,19 +129,19 @@ export class SubprocessService {
     const port = (await SubprocessService.freePort()).toString();
     console.debug(
       this.cmd,
-      this.args.map((arg) => arg.replace("{{PORT}}", port))
+      this.args.map((arg) => arg.replace("{{PORT}}", port)),
     );
     this.process = childProcess.spawn(
       this.cmd,
       this.args.map((arg) => arg.replace("{{PORT}}", port)),
       {
         stdio: "inherit",
-      }
+      },
     );
 
     try {
       console.debug(
-        `Waiting for ${this.name} to be available on port ${port}.`
+        `Waiting for ${this.name} to be available on port ${port}.`,
       );
       await this.portReady(port, host, 10000);
       console.debug(`Service ${this.name} available.`);
@@ -195,7 +195,7 @@ export class SubprocessService {
     }
     if (!connected) {
       throw new Error(
-        "Timed out waiting for service after " + timeoutMs + "ms."
+        "Timed out waiting for service after " + timeoutMs + "ms.",
       );
     }
   }
@@ -203,7 +203,7 @@ export class SubprocessService {
 
 export function serviceProviderFromJavaGradleTarget(
   gradleTarget: string,
-  args: string[] | undefined = undefined
+  args: string[] | undefined = undefined,
 ): () => Promise<Service> {
   return async () => {
     let jar: string;
@@ -216,7 +216,7 @@ export function serviceProviderFromJavaGradleTarget(
       }
     } else {
       jar = await JavaJarService.cachedJar(
-        await JavaJarService.gradleToJar(gradleTarget)
+        await JavaJarService.gradleToJar(gradleTarget),
       );
     }
 
@@ -234,7 +234,7 @@ export class JavaJarService extends SubprocessService {
   constructor(
     jar: string,
     args: string[] | undefined = undefined,
-    name: string | undefined = undefined
+    name: string | undefined = undefined,
   ) {
     if (!args) {
       // TODO: (Extension) Should filesToStage be set at some higher level?
@@ -245,13 +245,13 @@ export class JavaJarService extends SubprocessService {
 
   static async cachedJar(
     urlOrPath: string,
-    cacheDir: string = JavaJarService.JAR_CACHE
+    cacheDir: string = JavaJarService.JAR_CACHE,
   ): Promise<string> {
     if (urlOrPath.match(/^https?:\/\//)) {
       fs.mkdirSync(cacheDir, { recursive: true });
       const dest = path.join(
         JavaJarService.JAR_CACHE,
-        path.basename(urlOrPath)
+        path.basename(urlOrPath),
       );
       if (fs.existsSync(dest)) {
         return dest;
@@ -264,7 +264,7 @@ export class JavaJarService extends SubprocessService {
         const request = https.get(urlOrPath, function (response) {
           if (response.statusCode !== 200) {
             reject(
-              `Error code ${response.statusCode} when downloading ${urlOrPath}`
+              `Error code ${response.statusCode} when downloading ${urlOrPath}`,
             );
           }
           response.pipe(fout);
@@ -284,7 +284,7 @@ export class JavaJarService extends SubprocessService {
   static async gradleToJar(
     gradleTarget: string,
     appendix: string | undefined = undefined,
-    version: string = beamVersion
+    version: string = beamVersion,
   ): Promise<string> {
     if (version.startsWith("0.")) {
       // node-ts 0.x corresponds to Beam 2.x.
@@ -304,8 +304,8 @@ export class JavaJarService extends SubprocessService {
             artifactId,
             version.replace("-SNAPSHOT", ""),
             "SNAPSHOT",
-            appendix
-          )
+            appendix,
+          ),
         );
 
     if (version.includes("SNAPSHOT") && !projectRoot) {
@@ -318,14 +318,14 @@ export class JavaJarService extends SubprocessService {
     } else if (version.includes("SNAPSHOT")) {
       throw new Error(
         `${localPath} not found. Please build the server with
-      cd ${projectRoot}; ./gradlew ${gradleTarget})`
+      cd ${projectRoot}; ./gradlew ${gradleTarget})`,
       );
     } else {
       return JavaJarService.mavenJarUrl(
         artifactId,
         version,
         undefined,
-        appendix
+        appendix,
       );
     }
   }
@@ -336,7 +336,7 @@ export class JavaJarService extends SubprocessService {
     classifier: string | undefined = undefined,
     appendix: string | undefined = undefined,
     repo: string = JavaJarService.APACHE_REPOSITORY,
-    groupId: string = JavaJarService.BEAM_GROUP_ID
+    groupId: string = JavaJarService.BEAM_GROUP_ID,
   ): Promise<string> {
     if (version == "latest") {
       const medatadataUrl = [
@@ -374,7 +374,7 @@ export class JavaJarService extends SubprocessService {
     artifactId: string,
     version: string,
     classifier: string | undefined,
-    appendix: string | undefined
+    appendix: string | undefined,
   ): string {
     return (
       [artifactId, appendix, version, classifier]
@@ -408,13 +408,13 @@ export class PythonService extends SubprocessService {
       "..",
       "..",
       "resources",
-      "bootstrap_beam_venv.py"
+      "bootstrap_beam_venv.py",
     );
     console.debug("Invoking Python bootstrap script.");
     const result = childProcess.spawnSync(
       PythonService.whichPython(),
       [bootstrapScript],
-      { encoding: "latin1" }
+      { encoding: "latin1" },
     );
     if (result.status === 0) {
       console.debug(result.stdout);
@@ -446,7 +446,7 @@ export class PythonService extends SubprocessService {
   private constructor(
     pythonExecutablePath: string,
     module: string,
-    args: string[] = []
+    args: string[] = [],
   ) {
     super(pythonExecutablePath, ["-u", "-m", module].concat(args), module);
   }
@@ -482,7 +482,7 @@ function getBeamProjectRoot(): string | undefined {
     const projectRoot = path.dirname(findGitRoot(__dirname));
     if (
       fs.existsSync(
-        path.join(projectRoot, "sdks", "typescript", "src", "apache_beam")
+        path.join(projectRoot, "sdks", "typescript", "src", "apache_beam"),
       )
     ) {
       return projectRoot;
diff --git a/sdks/typescript/src/apache_beam/utils/utils.ts b/sdks/typescript/src/apache_beam/utils/utils.ts
index d89b7c365846..ba5b28d94f4b 100644
--- a/sdks/typescript/src/apache_beam/utils/utils.ts
+++ b/sdks/typescript/src/apache_beam/utils/utils.ts
@@ -28,6 +28,6 @@ export function camelToSnakeOptions(options: { [key: string]: any }): {
   return Object.fromEntries(
     Object.entries(options)
       .filter(([k, v]) => v != undefined)
-      .map(([k, v]) => [camelToSnake(k), v])
+      .map(([k, v]) => [camelToSnake(k), v]),
   );
 }
diff --git a/sdks/typescript/src/apache_beam/values.ts b/sdks/typescript/src/apache_beam/values.ts
index b484eb3eec85..a97185002c8c 100644
--- a/sdks/typescript/src/apache_beam/values.ts
+++ b/sdks/typescript/src/apache_beam/values.ts
@@ -36,7 +36,10 @@ export class GlobalWindow implements Window {
 }
 
 export class IntervalWindow implements Window {
-  constructor(public start: Instant, public end: Instant) {}
+  constructor(
+    public start: Instant,
+    public end: Instant,
+  ) {}
 
   maxTimestamp() {
     return this.end.sub(1);
diff --git a/sdks/typescript/src/apache_beam/worker/data.ts b/sdks/typescript/src/apache_beam/worker/data.ts
index 53a98ac4cc16..1c973d0914c8 100644
--- a/sdks/typescript/src/apache_beam/worker/data.ts
+++ b/sdks/typescript/src/apache_beam/worker/data.ts
@@ -41,7 +41,7 @@ export class MultiplexingDataChannel {
       endpoint,
       grpc.ChannelCredentials.createInsecure(),
       {},
-      {}
+      {},
     );
     this.dataChannel = this.dataClient.data(metadata);
     this.dataChannel.on("data", async (elements) => {
@@ -59,7 +59,7 @@ export class MultiplexingDataChannel {
       for (const timers of elements.timers) {
         const consumer = this.getConsumer(
           timers.instructionId,
-          timers.transformId
+          timers.transformId,
         );
         try {
           await consumer.sendTimers(timers.timerFamilyId, timers.timers);
@@ -83,7 +83,7 @@ export class MultiplexingDataChannel {
   async registerConsumer(
     bundleId: string,
     transformId: string,
-    consumer: IDataChannel
+    consumer: IDataChannel,
   ) {
     consumer = truncateOnErrorDataChannel(consumer);
     if (!this.consumers.has(bundleId)) {
diff --git a/sdks/typescript/src/apache_beam/worker/external_worker_service.ts b/sdks/typescript/src/apache_beam/worker/external_worker_service.ts
index 176c63f1ece4..cd29a238ec67 100644
--- a/sdks/typescript/src/apache_beam/worker/external_worker_service.ts
+++ b/sdks/typescript/src/apache_beam/worker/external_worker_service.ts
@@ -49,7 +49,7 @@ export class ExternalWorkerPool {
     const workerService: IBeamFnExternalWorkerPool = {
       startWorker(
         call: grpc.ServerUnaryCall<StartWorkerRequest, StartWorkerResponse>,
-        callback: grpc.sendUnaryData<StartWorkerResponse>
+        callback: grpc.sendUnaryData<StartWorkerResponse>,
       ): void {
         call.on("error", (args) => {
           console.error("unary() got error:", args);
@@ -62,8 +62,8 @@ export class ExternalWorkerPool {
             {
               controlUrl: call.request?.controlEndpoint?.url!,
             },
-            {}
-          )
+            {},
+          ),
         );
         callback(null, {
           error: "",
@@ -72,7 +72,7 @@ export class ExternalWorkerPool {
 
       stopWorker(
         call: grpc.ServerUnaryCall<StopWorkerRequest, StopWorkerResponse>,
-        callback: grpc.sendUnaryData<StopWorkerResponse>
+        callback: grpc.sendUnaryData<StopWorkerResponse>,
       ): void {
         this_.workers.get(call.request.workerId)?.stop();
         this_.workers.delete(call.request.workerId);
@@ -98,7 +98,7 @@ export class ExternalWorkerPool {
             this_.server.start();
             resolve(this_.address);
           }
-        }
+        },
       );
     });
   }
diff --git a/sdks/typescript/src/apache_beam/worker/logging.ts b/sdks/typescript/src/apache_beam/worker/logging.ts
index 0a7441ae88ca..14a1b542c3d2 100644
--- a/sdks/typescript/src/apache_beam/worker/logging.ts
+++ b/sdks/typescript/src/apache_beam/worker/logging.ts
@@ -124,10 +124,10 @@ export function createLoggingChannel(workerId: string, endpoint: string) {
   }
 
   startCapture(process.stdout, (out) =>
-    logQueue.enqueue(toEntry(out, guessLogLevel(out)))
+    logQueue.enqueue(toEntry(out, guessLogLevel(out))),
   );
   startCapture(process.stderr, (out) =>
-    logQueue.enqueue(toEntry(out, guessLogLevel(out)))
+    logQueue.enqueue(toEntry(out, guessLogLevel(out))),
   );
   const metadata = new grpc.Metadata();
   metadata.add("worker_id", workerId);
@@ -135,7 +135,7 @@ export function createLoggingChannel(workerId: string, endpoint: string) {
     endpoint,
     grpc.ChannelCredentials.createInsecure(),
     {},
-    {}
+    {},
   );
   const channel = client.logging(metadata);
 
diff --git a/sdks/typescript/src/apache_beam/worker/metrics.ts b/sdks/typescript/src/apache_beam/worker/metrics.ts
index 1a12479df685..3c0d991ac48c 100644
--- a/sdks/typescript/src/apache_beam/worker/metrics.ts
+++ b/sdks/typescript/src/apache_beam/worker/metrics.ts
@@ -174,7 +174,7 @@ class Distribution implements MetricCell<number> {
 
 metricTypes.set(
   "beam:metric:user:distribution_int64:v1",
-  (spec) => new Distribution(spec)
+  (spec) => new Distribution(spec),
 );
 
 class ElementCount extends Counter {
@@ -188,7 +188,7 @@ class ElementCount extends Counter {
 
 metricTypes.set(
   "beam:metric:user:element_count:v1",
-  (spec) => new ElementCount(spec)
+  (spec) => new ElementCount(spec),
 );
 
 /**
@@ -202,7 +202,7 @@ class ScopedMetricCell<T> {
     public value: MetricCell<unknown>,
     public name?: string,
     public transformId?: string,
-    public pcollectionId?: string
+    public pcollectionId?: string,
   ) {
     this.is_set = false;
   }
@@ -243,7 +243,7 @@ export class MetricsContainer {
   getMetric(
     transformId: string | undefined,
     pcollectionId: string | undefined,
-    spec: MetricSpec
+    spec: MetricSpec,
   ): ScopedMetricCell<unknown> {
     const key =
       spec.metricType +
@@ -257,7 +257,7 @@ export class MetricsContainer {
         createMetric(spec),
         spec.name,
         transformId,
-        pcollectionId
+        pcollectionId,
       );
       this.metrics.set(key, cell);
     }
@@ -278,7 +278,7 @@ export class MetricsContainer {
         .map((metric) => [
           shortIdCache.getShortId(metric),
           metric.value.payload(),
-        ])
+        ]),
     );
   }
 
@@ -313,7 +313,7 @@ export class MetricsShortIdCache {
 
   asMonitoringInfo(
     id: string,
-    payload: Uint8Array | undefined = undefined
+    payload: Uint8Array | undefined = undefined,
   ): MonitoringInfo {
     const result = this.idToInfo.get(id);
     if (payload !== undefined) {
@@ -326,7 +326,7 @@ export class MetricsShortIdCache {
 
 export function aggregateMetrics(
   infos: MonitoringInfo[],
-  urn: string
+  urn: string,
 ): Map<string, any> {
   const cells = new Map<string, MetricCell<any>>();
   for (const info of infos) {
@@ -342,7 +342,7 @@ export function aggregateMetrics(
     }
   }
   return new Map<string, any>(
-    Array.from(cells.entries()).map(([name, cell]) => [name, cell.extract()])
+    Array.from(cells.entries()).map(([name, cell]) => [name, cell.extract()]),
   );
 }
 
diff --git a/sdks/typescript/src/apache_beam/worker/operators.ts b/sdks/typescript/src/apache_beam/worker/operators.ts
index 6338fc0dab58..941ad62ef2d5 100644
--- a/sdks/typescript/src/apache_beam/worker/operators.ts
+++ b/sdks/typescript/src/apache_beam/worker/operators.ts
@@ -80,7 +80,7 @@ export class Receiver {
   constructor(
     private operators: IOperator[],
     private loggingStageInfo: LoggingStageInfo,
-    private elementCounter: { update: (number) => void }
+    private elementCounter: { update: (number) => void },
   ) {}
 
   receive(wvalue: WindowedValue<unknown>): ProcessResult {
@@ -113,7 +113,7 @@ export class OperatorContext {
     public getStateProvider: () => StateProvider,
     public getBundleId: () => string,
     public loggingStageInfo: LoggingStageInfo,
-    public metricsContainer: MetricsContainer
+    public metricsContainer: MetricsContainer,
   ) {
     this.pipelineContext = new PipelineContext(descriptor, "");
   }
@@ -121,7 +121,7 @@ export class OperatorContext {
 
 export function createOperator(
   transformId: string,
-  context: OperatorContext
+  context: OperatorContext,
 ): IOperator {
   const transform = context.descriptor.transforms[transformId];
   // Ensure receivers are eagerly created.
@@ -136,13 +136,13 @@ export function createOperator(
 type OperatorConstructor = (
   transformId: string,
   transformProto: PTransform,
-  context: OperatorContext
+  context: OperatorContext,
 ) => IOperator;
 interface OperatorClass {
   new (
     transformId: string,
     transformProto: PTransform,
-    context: OperatorContext
+    context: OperatorContext,
   ): IOperator;
 }
 
@@ -156,7 +156,7 @@ export function registerOperator(urn: string, cls: OperatorClass) {
 
 export function registerOperatorConstructor(
   urn: string,
-  constructor: OperatorConstructor
+  constructor: OperatorConstructor,
 ) {
   operatorsByUrn.set(urn, constructor);
 }
@@ -182,16 +182,16 @@ export class DataSourceOperator implements IOperator {
   constructor(
     transformId: string,
     transform: PTransform,
-    context: OperatorContext
+    context: OperatorContext,
   ) {
     const readPort = RemoteGrpcPort.fromBinary(transform.spec!.payload);
     this.multiplexingDataChannel = context.getDataChannel(
-      readPort.apiServiceDescriptor!.url
+      readPort.apiServiceDescriptor!.url,
     );
     this.transformId = transformId;
     this.getBundleId = context.getBundleId;
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
     this.coder = context.pipelineContext.getCoder(readPort.coderId);
     this.loggingStageInfo = context.loggingStageInfo;
@@ -225,7 +225,7 @@ export class DataSourceOperator implements IOperator {
             }
             this_.lastProcessedElement += 1;
             const maybePromise = this_.receiver.receive(
-              this_.coder.decode(reader, CoderContext.needsDelimiters)
+              this_.coder.decode(reader, CoderContext.needsDelimiters),
             );
             if (maybePromise !== NonPromise) {
               await maybePromise;
@@ -249,7 +249,7 @@ export class DataSourceOperator implements IOperator {
         onError: function (error: Error) {
           endOfDataReject(error);
         },
-      }
+      },
     );
   }
 
@@ -258,7 +258,7 @@ export class DataSourceOperator implements IOperator {
   }
 
   split(
-    desiredSplit: fnApi.ProcessBundleSplitRequest_DesiredSplit
+    desiredSplit: fnApi.ProcessBundleSplitRequest_DesiredSplit,
   ): fnApi.ProcessBundleSplitResponse_ChannelSplit | undefined {
     if (!this.started) {
       return undefined;
@@ -277,7 +277,7 @@ export class DataSourceOperator implements IOperator {
     // the end.
     var targetLastToProcessElement = Math.floor(
       this.lastProcessedElement +
-        (end - this.lastProcessedElement) * desiredSplit.fractionOfRemainder
+        (end - this.lastProcessedElement) * desiredSplit.fractionOfRemainder,
     );
     // If desiredSplit.allowedSplitPoints is populated, try to find the closest
     // split point that's in this list.
@@ -287,9 +287,9 @@ export class DataSourceOperator implements IOperator {
           ...Array.from(desiredSplit.allowedSplitPoints)
             .filter(
               (allowedSplitPoint) =>
-                allowedSplitPoint >= targetLastToProcessElement + 1
+                allowedSplitPoint >= targetLastToProcessElement + 1,
             )
-            .map(Number)
+            .map(Number),
         ) - 1;
     }
     // If we were able to find a valid, meaningful split point, record it
@@ -315,7 +315,7 @@ export class DataSourceOperator implements IOperator {
     } finally {
       this.multiplexingDataChannel.unregisterConsumer(
         this.getBundleId(),
-        this.transformId
+        this.transformId,
       );
       this.started = false;
     }
@@ -335,11 +335,11 @@ class DataSinkOperator implements IOperator {
   constructor(
     transformId: string,
     transform: PTransform,
-    context: OperatorContext
+    context: OperatorContext,
   ) {
     const writePort = RemoteGrpcPort.fromBinary(transform.spec!.payload);
     this.multiplexingDataChannel = context.getDataChannel(
-      writePort.apiServiceDescriptor!.url
+      writePort.apiServiceDescriptor!.url,
     );
     this.transformId = transformId;
     this.getBundleId = context.getBundleId;
@@ -349,7 +349,7 @@ class DataSinkOperator implements IOperator {
   async startBundle() {
     this.channel = this.multiplexingDataChannel.getSendChannel(
       this.getBundleId(),
-      this.transformId
+      this.transformId,
     );
     this.buffer = new protobufjs.Writer();
   }
@@ -383,10 +383,10 @@ class FlattenOperator implements IOperator {
   constructor(
     public transformId: string,
     transform: PTransform,
-    context: OperatorContext
+    context: OperatorContext,
   ) {
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
   }
 
@@ -410,10 +410,10 @@ abstract class CombineOperator<I, A, O> {
   constructor(
     public transformId: string,
     transform: PTransform,
-    context: OperatorContext
+    context: OperatorContext,
   ) {
     this.receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
     const spec = runnerApi.CombinePayload.fromBinary(transform.spec!.payload);
     this.combineFn = deserializeFn(spec.combineFn!.payload).combineFn;
@@ -431,7 +431,7 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
   maxKeys: number = 10000;
 
   static checkSupportsWindowing(
-    windowingStrategy: runnerApi.WindowingStrategy
+    windowingStrategy: runnerApi.WindowingStrategy,
   ) {
     if (
       windowingStrategy.mergeStatus !== runnerApi.MergeStatus_Enum.NON_MERGING
@@ -442,7 +442,7 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
       windowingStrategy.outputTime !== runnerApi.OutputTime_Enum.END_OF_WINDOW
     ) {
       throw new Error(
-        "Unsupported windowing output time: " + windowingStrategy
+        "Unsupported windowing output time: " + windowingStrategy,
       );
     }
   }
@@ -450,7 +450,7 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
   constructor(
     transformId: string,
     transform: PTransform,
-    context: OperatorContext
+    context: OperatorContext,
   ) {
     super(transformId, transform, context);
     const inputPc =
@@ -458,13 +458,13 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
         onlyElement(Object.values(transform.inputs))
       ];
     this.keyCoder = context.pipelineContext.getCoder(
-      context.descriptor.coders[inputPc.coderId].componentCoderIds[0]
+      context.descriptor.coders[inputPc.coderId].componentCoderIds[0],
     );
     const windowingStrategy =
       context.descriptor.windowingStrategies[inputPc.windowingStrategyId];
     CombinePerKeyPrecombineOperator.checkSupportsWindowing(windowingStrategy);
     this.windowCoder = context.pipelineContext.getCoder(
-      windowingStrategy.windowCoderId
+      windowingStrategy.windowCoderId,
     );
   }
 
@@ -479,7 +479,7 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
       }
       this.groups.set(
         wkey,
-        this.combineFn.addInput(this.groups.get(wkey), wvalue.value.value)
+        this.combineFn.addInput(this.groups.get(wkey), wvalue.value.value),
       );
     }
     if (this.groups.size > this.maxKeys) {
@@ -512,7 +512,7 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
           windows: [window],
           timestamp: window.maxTimestamp(),
           pane: PaneInfoCoder.ONE_AND_ONLY_FIRING,
-        })
+        }),
       );
       toDelete.push(wkey);
       if (this.groups.size - toDelete.length <= target) {
@@ -536,7 +536,7 @@ export class CombinePerKeyPrecombineOperator<I, A, O>
 
 registerOperator(
   "beam:transform:combine_per_key_precombine:v1",
-  CombinePerKeyPrecombineOperator
+  CombinePerKeyPrecombineOperator,
 );
 
 class CombinePerKeyMergeAccumulatorsOperator<I, A, O>
@@ -560,7 +560,7 @@ class CombinePerKeyMergeAccumulatorsOperator<I, A, O>
 
 registerOperator(
   "beam:transform:combine_per_key_merge_accumulators:v1",
-  CombinePerKeyMergeAccumulatorsOperator
+  CombinePerKeyMergeAccumulatorsOperator,
 );
 
 class CombinePerKeyExtractOutputsOperator<I, A, O>
@@ -584,7 +584,7 @@ class CombinePerKeyExtractOutputsOperator<I, A, O>
 
 registerOperator(
   "beam:transform:combine_per_key_extract_outputs:v1",
-  CombinePerKeyExtractOutputsOperator
+  CombinePerKeyExtractOutputsOperator,
 );
 
 class CombinePerKeyConvertToAccumulatorsOperator<I, A, O>
@@ -600,7 +600,7 @@ class CombinePerKeyConvertToAccumulatorsOperator<I, A, O>
         key,
         value: this.combineFn.addInput(
           this.combineFn.createAccumulator(),
-          value
+          value,
         ),
       },
       windows: wvalue.windows,
@@ -614,7 +614,7 @@ class CombinePerKeyConvertToAccumulatorsOperator<I, A, O>
 
 registerOperator(
   "beam:transform:combine_per_key_convert_to_accumulators:v1",
-  CombinePerKeyConvertToAccumulatorsOperator
+  CombinePerKeyConvertToAccumulatorsOperator,
 );
 
 class CombinePerKeyCombineGroupedValuesOperator<I, A, O>
@@ -645,7 +645,7 @@ class CombinePerKeyCombineGroupedValuesOperator<I, A, O>
 
 registerOperator(
   "beam:transform:combine_grouped_values:v1",
-  CombinePerKeyCombineGroupedValuesOperator
+  CombinePerKeyCombineGroupedValuesOperator,
 );
 
 // ParDo operators.
@@ -668,7 +668,7 @@ class GenericParDoOperator implements IOperator {
       context: any;
     },
     transformProto: runnerApi.PTransform,
-    operatorContext: OperatorContext
+    operatorContext: OperatorContext,
   ) {
     this.doFn = payload.doFn;
     this.originalContext = payload.context;
@@ -676,7 +676,7 @@ class GenericParDoOperator implements IOperator {
     this.sideInputInfo = createSideInputInfo(
       transformProto,
       spec,
-      operatorContext
+      operatorContext,
     );
     this.metricsContainer = operatorContext.metricsContainer;
   }
@@ -686,10 +686,10 @@ class GenericParDoOperator implements IOperator {
       this.transformId,
       this.sideInputInfo,
       this.getStateProvider,
-      this.metricsContainer
+      this.metricsContainer,
     );
     this.augmentedContext = this.paramProvider.augmentContext(
-      this.originalContext
+      this.originalContext,
     );
     if (this.doFn.startBundle) {
       this.doFn.startBundle(this.augmentedContext);
@@ -709,7 +709,7 @@ class GenericParDoOperator implements IOperator {
             windows: [window],
             pane: wvalue.pane,
             timestamp: wvalue.timestamp,
-          })
+          }),
         );
       }
       return result.build();
@@ -719,7 +719,7 @@ class GenericParDoOperator implements IOperator {
     function reallyProcess(): ProcessResult {
       const doFnOutput = this_.doFn.process(
         wvalue.value,
-        this_.augmentedContext
+        this_.augmentedContext,
       );
       if (!doFnOutput) {
         return NonPromise;
@@ -732,7 +732,7 @@ class GenericParDoOperator implements IOperator {
             windows: wvalue.windows,
             pane: wvalue.pane,
             timestamp: wvalue.timestamp,
-          })
+          }),
         );
       }
       this_.paramProvider.setCurrentValue(undefined);
@@ -780,7 +780,10 @@ class GenericParDoOperator implements IOperator {
 }
 
 class IdentityParDoOperator implements IOperator {
-  constructor(public transformId: string, private receiver: Receiver) {}
+  constructor(
+    public transformId: string,
+    private receiver: Receiver,
+  ) {}
 
   async startBundle() {}
 
@@ -795,7 +798,7 @@ class SplittingDoFnOperator implements IOperator {
   constructor(
     public transformId: string,
     private receivers: { [key: string]: Receiver },
-    private options: SplitOptions
+    private options: SplitOptions,
   ) {}
 
   async startBundle() {}
@@ -805,7 +808,7 @@ class SplittingDoFnOperator implements IOperator {
     const keys = Object.keys(wvalue.value as object);
     if (this.options.exclusive && keys.length !== 1) {
       throw new Error(
-        "Multiple keys for exclusively split element: " + wvalue.value
+        "Multiple keys for exclusively split element: " + wvalue.value,
       );
     }
     for (let tag of keys) {
@@ -821,7 +824,7 @@ class SplittingDoFnOperator implements IOperator {
               "' for " +
               wvalue.value +
               " not in " +
-              this.options.knownTags
+              this.options.knownTags,
           );
         }
       }
@@ -833,7 +836,7 @@ class SplittingDoFnOperator implements IOperator {
             windows: wvalue.windows,
             timestamp: wvalue.timestamp,
             pane: wvalue.pane,
-          })
+          }),
         );
       }
     }
@@ -847,7 +850,7 @@ class AssignWindowsParDoOperator implements IOperator {
   constructor(
     public transformId: string,
     private receiver: Receiver,
-    private windowFn: WindowFn<Window>
+    private windowFn: WindowFn<Window>,
   ) {}
 
   async startBundle() {}
@@ -878,7 +881,7 @@ class AssignTimestampsParDoOperator implements IOperator {
   constructor(
     public transformId: string,
     private receiver: Receiver,
-    private func: (any, Instant) => typeof Instant
+    private func: (any, Instant) => typeof Instant,
   ) {}
 
   async startBundle() {}
@@ -900,7 +903,7 @@ registerOperatorConstructor(
   parDo.urn,
   (transformId: string, transform: PTransform, context: OperatorContext) => {
     const receiver = context.getReceiver(
-      onlyElement(Object.values(transform.outputs))
+      onlyElement(Object.values(transform.outputs)),
     );
     const spec = runnerApi.ParDoPayload.fromBinary(transform.spec!.payload);
     // TODO: (Cleanup) Ideally we could branch on the urn itself, but some runners have a closed set of known URNs.
@@ -911,24 +914,24 @@ registerOperatorConstructor(
         spec,
         deserializeFn(spec.doFn.payload!),
         transform,
-        context
+        context,
       );
     } else if (spec.doFn?.urn === urns.IDENTITY_DOFN_URN) {
       return new IdentityParDoOperator(
         transformId,
-        context.getReceiver(onlyElement(Object.values(transform.outputs)))
+        context.getReceiver(onlyElement(Object.values(transform.outputs))),
       );
     } else if (spec.doFn?.urn === urns.JS_WINDOW_INTO_DOFN_URN) {
       return new AssignWindowsParDoOperator(
         transformId,
         context.getReceiver(onlyElement(Object.values(transform.outputs))),
-        deserializeFn(spec.doFn.payload!).windowFn
+        deserializeFn(spec.doFn.payload!).windowFn,
       );
     } else if (spec.doFn?.urn === urns.JS_ASSIGN_TIMESTAMPS_DOFN_URN) {
       return new AssignTimestampsParDoOperator(
         transformId,
         context.getReceiver(onlyElement(Object.values(transform.outputs))),
-        deserializeFn(spec.doFn.payload!).func
+        deserializeFn(spec.doFn.payload!).func,
       );
     } else if (spec.doFn?.urn === urns.SPLITTING_JS_DOFN_URN) {
       return new SplittingDoFnOperator(
@@ -937,14 +940,14 @@ registerOperatorConstructor(
           Object.entries(transform.outputs).map(([tag, pcId]) => [
             tag,
             context.getReceiver(pcId),
-          ])
+          ]),
         ),
-        deserializeFn(spec.doFn.payload!)
+        deserializeFn(spec.doFn.payload!),
       );
     } else {
       throw new Error("Unknown DoFn type: " + spec);
     }
-  }
+  },
 );
 
 ///
@@ -958,7 +961,7 @@ export function encodeToBase64<T>(element: T, coder: Coder<T>): string {
 export function decodeFromBase64<T>(s: string, coder: Coder<T>): T {
   return coder.decode(
     new protobufjs.Reader(Buffer.from(s, "base64")),
-    CoderContext.wholeStream
+    CoderContext.wholeStream,
   );
 }
 
diff --git a/sdks/typescript/src/apache_beam/worker/pardo_context.ts b/sdks/typescript/src/apache_beam/worker/pardo_context.ts
index f60c103d2650..62e04751075e 100644
--- a/sdks/typescript/src/apache_beam/worker/pardo_context.ts
+++ b/sdks/typescript/src/apache_beam/worker/pardo_context.ts
@@ -58,7 +58,7 @@ export class ParamProviderImpl implements ParamProvider {
     private transformId: string,
     private sideInputInfo: Map<string, SideInputInfo>,
     private getStateProvider: () => StateProvider,
-    private metricsContainer: MetricsContainer
+    private metricsContainer: MetricsContainer,
   ) {}
 
   // Avoid modifying the original object, as that could have surprising results
@@ -82,8 +82,8 @@ export class ParamProviderImpl implements ParamProvider {
         if ((value as ParDoParam).parDoParamName === "sideInput") {
           this.prefetchCallbacks.push(
             this.prefetchSideInput(
-              value as SideInputParam<unknown, unknown, unknown>
-            )
+              value as SideInputParam<unknown, unknown, unknown>,
+            ),
           );
         }
       }
@@ -92,7 +92,7 @@ export class ParamProviderImpl implements ParamProvider {
   }
 
   prefetchSideInput(
-    param: SideInputParam<unknown, unknown, unknown>
+    param: SideInputParam<unknown, unknown, unknown>,
   ): (window: Window) => operators.ProcessResult {
     const this_ = this;
     const stateProvider = this.getStateProvider();
@@ -106,7 +106,7 @@ export class ParamProviderImpl implements ParamProvider {
           while (reader.pos < reader.len) {
             yield elementCoder.decode(reader, CoderContext.needsDelimiters);
           }
-        })()
+        })(),
       );
     };
     return (window: Window) => {
@@ -118,7 +118,7 @@ export class ParamProviderImpl implements ParamProvider {
         param.accessor.accessPattern,
         param.sideInputId,
         window,
-        windowCoder
+        windowCoder,
       );
       const lookupResult = stateProvider.getState(stateKey, decode);
       if (lookupResult.type === "value") {
@@ -133,7 +133,7 @@ export class ParamProviderImpl implements ParamProvider {
   }
 
   setCurrentValue(
-    wvalue: WindowedValue<unknown> | undefined
+    wvalue: WindowedValue<unknown> | undefined,
   ): operators.ProcessResult {
     this.wvalue = wvalue;
     if (wvalue === null || wvalue === undefined) {
@@ -155,7 +155,7 @@ export class ParamProviderImpl implements ParamProvider {
   lookup(param) {
     if (this.wvalue === null || this.wvalue === undefined) {
       throw new Error(
-        param.parDoParamName + " not defined outside of a process() call."
+        param.parDoParamName + " not defined outside of a process() call.",
       );
     }
 
@@ -201,7 +201,7 @@ export interface SideInputInfo {
 export function createSideInputInfo(
   transformProto: runnerApi.PTransform,
   spec: runnerApi.ParDoPayload,
-  operatorContext: operators.OperatorContext
+  operatorContext: operators.OperatorContext,
 ): Map<string, SideInputInfo> {
   const globalWindow = new GlobalWindow();
   const sideInputInfo: Map<string, SideInputInfo> = new Map();
@@ -216,7 +216,7 @@ export function createSideInputInfo(
         break;
       default:
         throw new Error(
-          "Unsupported window mapping fn: " + sideInput.windowMappingFn!.urn
+          "Unsupported window mapping fn: " + sideInput.windowMappingFn!.urn,
         );
     }
     const sidePColl =
@@ -225,12 +225,12 @@ export function createSideInputInfo(
       ];
     const windowingStrategy =
       operatorContext.pipelineContext.getWindowingStrategy(
-        sidePColl.windowingStrategyId
+        sidePColl.windowingStrategyId,
       );
     sideInputInfo.set(sideInputId, {
       elementCoder: operatorContext.pipelineContext.getCoder(sidePColl.coderId),
       windowCoder: operatorContext.pipelineContext.getCoder(
-        windowingStrategy.windowCoderId
+        windowingStrategy.windowCoderId,
       ),
       windowMappingFn: windowMappingFn,
     });
@@ -243,7 +243,7 @@ export function createStateKey(
   accessPattern: string,
   sideInputId: string,
   window: Window,
-  windowCoder: Coder<Window>
+  windowCoder: Coder<Window>,
 ): fnApi.StateKey {
   const writer = new protobufjs.Writer();
   windowCoder.encode(window, writer, CoderContext.needsDelimiters);
diff --git a/sdks/typescript/src/apache_beam/worker/state.ts b/sdks/typescript/src/apache_beam/worker/state.ts
index ad8493ade208..5a340cbb64f0 100644
--- a/sdks/typescript/src/apache_beam/worker/state.ts
+++ b/sdks/typescript/src/apache_beam/worker/state.ts
@@ -41,7 +41,7 @@ export type MaybePromise<T> = PromiseWrapper<T> | ValueWrapper<T>;
 export interface StateProvider {
   getState: <T>(
     stateKey: fnApi.StateKey,
-    decode: (data: Uint8Array) => T
+    decode: (data: Uint8Array) => T,
   ) => MaybePromise<T>;
 }
 
@@ -59,7 +59,7 @@ export class CachingStateProvider implements StateProvider {
     // serialized key, only constructing this proto when interacting with
     // the runner.
     const cacheKey = Buffer.from(fnApi.StateKey.toBinary(stateKey)).toString(
-      "base64"
+      "base64",
     );
     if (this.cache.has(cacheKey)) {
       return this.cache.get(cacheKey)!;
@@ -84,14 +84,14 @@ export class CachingStateProvider implements StateProvider {
 export class GrpcStateProvider implements StateProvider {
   constructor(
     private multiplexingChannel: MultiplexingStateChannel,
-    private instructionId
+    private instructionId,
   ) {}
 
   getState<T>(stateKey: fnApi.StateKey, decode: (data: Uint8Array) => T) {
     return this.multiplexingChannel.getState(
       this.instructionId,
       stateKey,
-      decode
+      decode,
     );
   }
 }
@@ -121,7 +121,7 @@ export class MultiplexingStateChannel {
       endpoint,
       grpc.ChannelCredentials.createInsecure(),
       {},
-      {}
+      {},
     );
     const metadata = new grpc.Metadata();
     metadata.add("worker_id", workerId);
@@ -145,7 +145,7 @@ export class MultiplexingStateChannel {
   getState<T>(
     instructionId: string,
     stateKey: fnApi.StateKey,
-    decode: (data: Uint8Array) => T
+    decode: (data: Uint8Array) => T,
   ): MaybePromise<T> {
     if (this.closed) {
       throw new Error("State stream is closed.");
@@ -160,7 +160,7 @@ export class MultiplexingStateChannel {
     function responseCallback(
       resolve,
       reject,
-      prevChunks: Uint8Array[] = []
+      prevChunks: Uint8Array[] = [],
     ): (response: fnApi.StateResponse) => void {
       return (response) => {
         if (this_.error) {
@@ -189,7 +189,7 @@ export class MultiplexingStateChannel {
           const continueId = "continueStateRequest" + this_.idCounter++;
           this_.callbacks.set(
             continueId,
-            responseCallback(resolve, reject, allChunks)
+            responseCallback(resolve, reject, allChunks),
           );
           this_.stateChannel.write({
             id: continueId,
@@ -235,7 +235,7 @@ export function Uint8ArrayConcat(chunks: Uint8Array[]) {
     return new Uint8Array();
   } else {
     const fullData = new Uint8Array(
-      chunks.map((chunk) => chunk.length).reduce((a, b) => a + b)
+      chunks.map((chunk) => chunk.length).reduce((a, b) => a + b),
     );
     let start = 0;
     for (const chunk of chunks) {
diff --git a/sdks/typescript/src/apache_beam/worker/worker.ts b/sdks/typescript/src/apache_beam/worker/worker.ts
index 0d7f2609f295..a68ae7100da2 100644
--- a/sdks/typescript/src/apache_beam/worker/worker.ts
+++ b/sdks/typescript/src/apache_beam/worker/worker.ts
@@ -80,7 +80,7 @@ export class Worker {
   constructor(
     private id: string,
     private endpoints: WorkerEndpoints,
-    options: Object = {}
+    options: Object = {},
   ) {
     const metadata = new grpc.Metadata();
     metadata.add("worker_id", this.id);
@@ -88,7 +88,7 @@ export class Worker {
       endpoints.controlUrl,
       grpc.ChannelCredentials.createInsecure(),
       {},
-      {}
+      {},
     );
     this.controlChannel = this.controlClient.control(metadata);
     this.controlChannel.on("data", this.handleRequest.bind(this));
@@ -152,7 +152,7 @@ export class Worker {
 
   async progress(request): Promise<InstructionResponse> {
     const processor = this.activeBundleProcessors.get(
-      request.request.processBundleProgress.instructionId
+      request.request.processBundleProgress.instructionId,
     );
     if (processor) {
       const monitoringData = processor.monitoringData(this.metricsShortIdCache);
@@ -164,7 +164,7 @@ export class Worker {
           processBundleProgress: {
             monitoringInfos: Array.from(monitoringData.entries()).map(
               ([id, payload]) =>
-                this.metricsShortIdCache.asMonitoringInfo(id, payload)
+                this.metricsShortIdCache.asMonitoringInfo(id, payload),
             ),
             monitoringData: Object.fromEntries(monitoringData.entries()),
           },
@@ -183,7 +183,7 @@ export class Worker {
 
   async split(request): Promise<InstructionResponse> {
     const processor = this.activeBundleProcessors.get(
-      request.request.processBundleSplit.instructionId
+      request.request.processBundleSplit.instructionId,
     );
     console.log(request.request.processBundleSplit, processor === undefined);
     if (processor) {
@@ -193,7 +193,7 @@ export class Worker {
         response: {
           oneofKind: "processBundleSplit",
           processBundleSplit: processor.split(
-            request.request.processBundleSplit
+            request.request.processBundleSplit,
           ),
         },
       };
@@ -237,11 +237,11 @@ export class Worker {
             } else {
               this.processBundleDescriptors.set(
                 descriptorId,
-                maybeStripDataflowWindowedWrappings(value)
+                maybeStripDataflowWindowedWrappings(value),
               );
               this.process(request);
             }
-          }
+          },
         );
         return;
       }
@@ -260,7 +260,7 @@ export class Worker {
             requiresFinalization: false,
             monitoringInfos: Array.from(monitoringData.entries()).map(
               ([id, payload]) =>
-                this.metricsShortIdCache.asMonitoringInfo(id, payload)
+                this.metricsShortIdCache.asMonitoringInfo(id, payload),
             ),
             monitoringData: Object.fromEntries(monitoringData.entries()),
           },
@@ -290,7 +290,7 @@ export class Worker {
       return new BundleProcessor(
         this.processBundleDescriptors.get(descriptorId)!,
         this.getDataChannel.bind(this),
-        this.getStateChannel.bind(this)
+        this.getStateChannel.bind(this),
       );
     }
   }
@@ -309,7 +309,7 @@ export class Worker {
     if (!this.dataChannels.has(endpoint)) {
       this.dataChannels.set(
         endpoint,
-        new MultiplexingDataChannel(endpoint, this.id)
+        new MultiplexingDataChannel(endpoint, this.id),
       );
     }
     return this.dataChannels.get(endpoint)!;
@@ -319,7 +319,7 @@ export class Worker {
     if (!this.stateChannels.has(endpoint)) {
       this.stateChannels.set(
         endpoint,
-        new MultiplexingStateChannel(endpoint, this.id)
+        new MultiplexingStateChannel(endpoint, this.id),
       );
     }
     return this.stateChannels.get(endpoint)!;
@@ -348,7 +348,7 @@ export class BundleProcessor {
     descriptor: ProcessBundleDescriptor,
     getDataChannel: (string) => MultiplexingDataChannel,
     getStateChannel: ((string) => MultiplexingStateChannel) | StateProvider,
-    root_urns = ["beam:runner:source:v1"]
+    root_urns = ["beam:runner:source:v1"],
   ) {
     this.descriptor = descriptor;
     this.getDataChannel = getDataChannel;
@@ -370,7 +370,7 @@ export class BundleProcessor {
             consumers.get(pcollectionId)!.push(transformId);
           });
         }
-      }
+      },
     );
 
     function getReceiver(pcollectionId: string): Receiver {
@@ -380,8 +380,8 @@ export class BundleProcessor {
           new Receiver(
             (consumers.get(pcollectionId) || []).map(getOperator),
             this_.loggingStageInfo,
-            this_.metricsContainer.elementCountMetric(pcollectionId)
-          )
+            this_.metricsContainer.elementCountMetric(pcollectionId),
+          ),
         );
       }
       return this_.receivers.get(pcollectionId)!;
@@ -400,9 +400,9 @@ export class BundleProcessor {
               this_.getStateProvider.bind(this_),
               this_.getBundleId.bind(this_),
               this_.loggingStageInfo,
-              this_.metricsContainer
-            )
-          )
+              this_.metricsContainer,
+            ),
+          ),
         );
         creationOrderedOperators.push(this_.operators.get(transformId)!);
       }
@@ -414,7 +414,7 @@ export class BundleProcessor {
         if (root_urns.includes(transform?.spec?.urn!)) {
           getOperator(transformId);
         }
-      }
+      },
     );
     this.topologicallyOrderedOperators = creationOrderedOperators.reverse();
   }
@@ -425,10 +425,10 @@ export class BundleProcessor {
         this.stateProvider = new CachingStateProvider(
           new GrpcStateProvider(
             this.getStateChannel(
-              this.descriptor.stateApiServiceDescriptor!.url
+              this.descriptor.stateApiServiceDescriptor!.url,
             ),
-            this.getBundleId()
-          )
+            this.getBundleId(),
+          ),
         );
       } else {
         this.stateProvider = this.getStateChannel;
@@ -474,7 +474,7 @@ export class BundleProcessor {
   split(splitRequest: ProcessBundleSplitRequest): ProcessBundleSplitResponse {
     const root = this.topologicallyOrderedOperators[0] as DataSourceOperator;
     for (const [target, desiredSplit] of Object.entries(
-      splitRequest.desiredSplits
+      splitRequest.desiredSplits,
     )) {
       if (target == root.transformId) {
         const channelSplit = root.split(desiredSplit);
@@ -510,7 +510,7 @@ function isPrimitive(transform: PTransform): boolean {
 }
 
 function maybeStripDataflowWindowedWrappings(
-  descriptor: ProcessBundleDescriptor
+  descriptor: ProcessBundleDescriptor,
 ): ProcessBundleDescriptor {
   for (const pcoll of Object.values(descriptor.pcollections)) {
     const coder = descriptor.coders[pcoll.coderId];
diff --git a/sdks/typescript/src/apache_beam/worker/worker_main.ts b/sdks/typescript/src/apache_beam/worker/worker_main.ts
index e70e056abb63..bc6c60c2efa6 100644
--- a/sdks/typescript/src/apache_beam/worker/worker_main.ts
+++ b/sdks/typescript/src/apache_beam/worker/worker_main.ts
@@ -48,7 +48,7 @@ async function main() {
       console.error(
         `**ERROR**
       Unable to require module '${m}' used in requireForSerialization:
-      please ensure that it is available in the package exports.`
+      please ensure that it is available in the package exports.`,
       );
       // Explicitly exit the process to avoid the error getting swallowed
       // by a long traceback.
@@ -62,7 +62,7 @@ async function main() {
     {
       controlUrl: argv.control_endpoint,
     },
-    options
+    options,
   );
   if (pushLogs) {
     await pushLogs();
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 3bd606327703..da170d0f8467 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -19,13 +19,13 @@ import com.gradle.enterprise.gradleplugin.internal.extension.BuildScanExtensionW
 
 pluginManagement {
   plugins {
-     id("org.javacc.javacc") version "3.0.0" // enable the JavaCC parser generator
+     id("org.javacc.javacc") version "3.0.2" // enable the JavaCC parser generator
   }
 }
 
 plugins {
   id("com.gradle.enterprise") version "3.15.1"
-  id("com.gradle.common-custom-user-data-gradle-plugin") version "1.11.3"
+  id("com.gradle.common-custom-user-data-gradle-plugin") version "1.12.1"
 }
 
 
@@ -86,6 +86,7 @@ include(":examples:multi-language")
 include(":learning")
 include(":learning:tour-of-beam")
 include(":learning:tour-of-beam:frontend")
+include(":learning:tour-of-beam:terraform")
 include(":model:fn-execution")
 include(":model:job-management")
 include(":model:pipeline")
@@ -115,16 +116,10 @@ include(":it:truthmatchers")
 include(":it:mongodb")
 include(":it:splunk")
 include(":it:neo4j")
-
-include(":learning:tour-of-beam:frontend")
-
 include(":runners:core-construction-java")
 include(":runners:core-java")
 include(":runners:direct-java")
 include(":runners:extensions-java:metrics")
-include(":learning")
-include(":learning:tour-of-beam")
-include(":learning:tour-of-beam:terraform")
 /* Begin Flink Runner related settings */
 // Flink 1.12
 include(":runners:flink:1.12")
@@ -317,7 +312,6 @@ include(":sdks:python:test-suites:tox:py311")
 include(":sdks:python:test-suites:xlang")
 include(":sdks:typescript")
 include(":sdks:typescript:container")
-include(":vendor:bytebuddy-1_12_8")
 include(":vendor:grpc-1_54_0")
 include(":vendor:calcite-1_28_0")
 include(":vendor:guava-32_1_2-jre")
diff --git a/vendor/grpc-1_54_0/build.gradle b/vendor/grpc-1_54_0/build.gradle
index 69e638fdff7c..6dbc9dbd08c4 100644
--- a/vendor/grpc-1_54_0/build.gradle
+++ b/vendor/grpc-1_54_0/build.gradle
@@ -23,7 +23,7 @@ plugins { id 'org.apache.beam.vendor-java' }
 description = "Apache Beam :: Vendored Dependencies :: gRPC :: 1.54.0"
 
 group = "org.apache.beam"
-version = "0.1"
+version = "0.2"
 
 vendorJava(
         dependencies: GrpcVendoring_1_54_0.dependencies(),
diff --git a/website/www/site/config.toml b/website/www/site/config.toml
index e1b9fd612ed7..df90aa617636 100644
--- a/website/www/site/config.toml
+++ b/website/www/site/config.toml
@@ -104,7 +104,7 @@ github_project_repo = "https://github.com/apache/beam"
 
 [params]
 description = "Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes."
-release_latest = "2.52.0"
+release_latest = "2.53.0"
 # The repository and branch where the files live in Github or Colab. This is used
 # to serve and stage from your local branch, but publish to the master branch.
 #   e.g. https://github.com/{{< param branch_repo >}}/path/to/notebook.ipynb
diff --git a/website/www/site/content/en/blog/apache-beam-flink-and-kubernetes-part2.md b/website/www/site/content/en/blog/apache-beam-flink-and-kubernetes-part2.md
new file mode 100644
index 000000000000..ceba71a4401d
--- /dev/null
+++ b/website/www/site/content/en/blog/apache-beam-flink-and-kubernetes-part2.md
@@ -0,0 +1,158 @@
+---
+title:  "Build a scalable, self-managed streaming infrastructure with Beam and Flink: Tackling Autoscaling Challenges - Part 2"
+date:   2023-12-18 09:00:00 -0400
+categories:
+  - blog
+authors:
+  - talat
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Build a scalable, self-managed streaming infrastructure with Flink: Tackling Autoscaling Challenges - Part 2
+
+
+Welcome to Part 2 of our in-depth series about building and managing a service for Apache Beam Flink on Kubernetes. In this segment, we're taking a closer look at the hurdles we encountered while implementing autoscaling. These challenges weren't just roadblocks. They were opportunities for us to innovate and enhance our system. Let’s break down these issues, understand their context, and explore the solutions we developed.
+
+## Understand Apache Beam backlog metrics in the Flink runner environment
+
+**The Challenge:** In our current setup, we are using Apache Flink for processing data streams. However, we've encountered a puzzling issue: our Flink job isn't showing the backlog metrics from Apache Beam. These metrics are critical for understanding the state and performance of our data pipelines.
+
+**What We Found:** Interestingly, we noticed that the metrics are actually being generated in `KafkaIO`, which is a part of our data pipeline that handles Kafka streams. But when we try to monitor these metrics through the Apache Flink Metric system, we can't find them. We suspected that there might be an issue with the integration (or 'wiring') between Apache Beam and Apache Flink.
+
+**Digging Deeper:** On closer inspection, we found that the metrics should be emitted during the 'Checkpointing' phase of the data stream processing. During this crucial step, the system takes a snapshot of the stream's state, and the metrics are typically metrics that are generated for unbounded sources. Unbounded sources are sources that continuously stream data, like Kafka.
+
+**A Potential Solution:** We believe the root of the problem lies in how the metric context is set during the checkpointing phase. A disconnect appears to prevent the Beam metrics from being properly captured in the Flink Metric system. We proposed a fix for this issue, which you can review and contribute to on our GitHub pull request: [Apache Beam PR #29793](https://github.com/apache/beam/pull/29793).
+
+
+<img class="center-block"
+src="/images/blog/apache-beam-flink-and-kubernetes-part2/flink-backlog-metrics.png"
+alt="Apache Flink Beam Backlog Metrics">
+
+
+## Overcoming challenges in checkpoint size reduction for autoscaling Beam jobs
+
+In this section we will discuss strategies for reducing the size of checkpoints in autoscaling Apache Beam jobs, focusing on efficient checkpointing in Apache Flink and optimizing bundle sizes and PipelineOptions to manage frequent checkpoint timeouts and large-scale job requirements.
+
+### Understand the basics of checkpointing in Apache Flink
+In stream processing, maintaining state consistency and fault tolerance is crucial. Apache Flink achieves this through a process called *checkpointing*. Checkpointing periodically captures the state of a job's operators and stores it in a stable storage location, like Google Cloud Storage or AWS S3. Specifically, Flink checkpoints a job every ten seconds and allows up to one minute for this process to complete. This process is vital for ensuring that, in case of failures, the job can resume from the last checkpoint, providing exactly-once semantics and fault tolerance.
+
+### The role of bundles in Apache Beam
+Apache Beam introduces the concept of a *bundle*. A bundle is essentially a group of elements that are processed together. This step enhances processing efficiency and throughput by reducing the overhead of handling each element separately. For more information, see [Bundling and persistence](https://beam.apache.org/documentation/runtime/model/#bundling-and-persistence). In the Flink runner [default configuration](https://beam.apache.org/releases/javadoc/2.52.0/org/apache/beam/runners/flink/FlinkPipelineOptions.html#getMaxBundleSize--), a bundle's default size is 1000 elements with a one-second timeout. However, based on our performance tests, we adjusted the bundle size to *10,000 elements with a 10-second timeout*.
+
+### Challenge: frequent checkpoint timeouts
+When we configured checkpointing every 10 seconds, we faced frequent checkpoint timeouts, often exceeding 1 minute. This was due to the large size of the checkpoints.
+
+### Solution: Manage checkpoint size
+In Apache Beam Flink jobs, the `finishBundleBeforeCheckpointing` option plays a pivotal role. When enabled, it ensures that all bundles are completely processed before initiating a checkpoint. This results in checkpoints that only contain the state post-bundle completion, significantly reducing checkpoint size. Initially, our checkpoints were around 2 MB per pipeline. With this change, they consistently dropped to 150 KB.
+
+### Address the checkpoint size in large-scale jobs
+Despite reducing checkpoint sizes, a 150 KB checkpoint every ten seconds can still be substantial, especially in jobs that run multiple pipelines. For instance, with 100 pipelines in a single job, this size balloons to 15 MB per 10-second interval.
+
+### Further optimization: reduce checkpoint size with PipelineOptions
+We discovered that due to a specific issue (BEAM-8577), our Flink runner was including our large `PipelineOptions` objects in every checkpoint. We solved this problem by removing unnecessary application-related options from `PipelineOptions`, further reducing the checkpoint size to a more manageable 10 KB per pipeline.
+
+
+## Kafka Reader wait time: solving autoscaling challenges in Beam jobs
+
+### Understand unaligned checkpointing
+
+In our system, we use unaligned checkpointing to speed up the process of checkpointing, which is essential for ensuring data consistency in distributed systems. However, when we activated the `finishBundleBeforeCheckpointing` feature, we began facing checkpoint timeout issues and delays in checkpointing steps. Apache Beam leverages Apache Flink's legacy source implementation for processing unbounded sources. In Flink, tasks are categorized into two types: source tasks and non-source tasks.
+
+- **Source tasks**: fetch data from external systems into a Flink job
+- **Non-source tasks**: process the incoming data
+
+<img class="center-block"
+src="/images/blog/apache-beam-flink-and-kubernetes-part2/kafkaio-wait-reader.png"
+alt="Apache Flink Task Types">
+
+In the standard configuration, non-source tasks check for an available buffer before pulling data. If source tasks don't perform this check, they might experience checkpointing delays in writing data to the output buffer. This delay affects the efficiency of unaligned checkpoints, which are only recognized by legacy source tasks when an output buffer is available.
+
+### Address the challenge with UnboundedSourceWrapper in Beam
+
+To solve this problem, Apache Flink introduced a new source implementation that operates in a pull mode. In this mode, a task checks for a free buffer before fetching data, aligning with the approach of non-source tasks.
+
+However, the legacy source, still used by Apache Beam's Flink Runner, operates in a push mode. It sends data to downstream tasks immediately. This setup might create bottlenecks when buffers are full, causing delays in detecting unaligned checkpoint barriers.
+
+### Our solution
+
+Despite its deprecation, Apache Beam's Flink Runner still uses the legacy source implementation. To address its issues, we implemented our modifications and the quick workarounds suggested in [FLINK-26759](https://issues.apache.org/jira/browse/FLINK-26759). These enhancements are detailed in our [Pull Request](#). You can also find more information about unaligned checkpoint issues in the [Flink Unaligned Checkpoint](https://blog.51cto.com/u_14286418/7000028) blog post.
+
+<img class="center-block"
+src="/images/blog/apache-beam-flink-and-kubernetes-part2/checkpoint_monitoring-history-subtasks.png"
+alt="Apache Flink UI Checkpoint History">
+
+## Address slow reads in high-traffic scenarios
+
+In our journey with Apache Beam and the Flink Runner, we encountered a significant challenge similar to one documented in the post [How Intuit Debug Consumer Lags in Apache Beam](https://antonio-si.medium.com/how-intuit-debug-consumer-lags-in-apache-beam-22ca3b39602e) by [Antonio Si](https://antonio-si.medium.com/) in his experience at Intuit. Their real-time data processing pipelines had increasing Kafka consumer lag, particularly with topics experiencing high message traffic. This issue was traced to Apache Beam's handling of Kafka partitions through `UnboundedSourceWrapper` and `KafkaUnboundedReader`. Specifically, for topics with lower traffic, the processing thread paused unnecessarily, delaying the processing of high-traffic topics. We faced a parallel situation in our system, where the imbalance in processing speeds between high- and low-traffic topics led to inefficiencies.
+
+<img class="center-block"
+src="/images/blog/apache-beam-flink-and-kubernetes-part2/adaptive-timeout-kafka.png"
+alt="UnboundedSourceWrapper Design">
+
+To resolve this issue, we developed an innovative solution: an adaptive timeout strategy in `KafkaIO`. This strategy dynamically adjusts the timeout duration based on the traffic of each topic. For low-traffic topics, it shortens the timeout, preventing unnecessary delays. For high-traffic topics, it extends the timeout, providing more processing opportunities. This approach is detailed in our recent pull request.
+
+## Unbalanced partition distribution in Beam job autoscaling
+
+At the heart of this system is the adaptive scheduler, a component designed for rapid resource allocation. It intelligently adjusts the number of parallel tasks (parallelism) a job performs based on the availability of computing slots. These slots are like individual workstations, each capable of handling certain parts of the job.
+
+However, we encountered a problem. Our jobs consist of multiple independent pipelines, each needing its own set of resources. Initially, the system tended to overburden the first few workers by assigning them more tasks, while others remained underutilized. This issue was due to the way Flink allocated tasks, favoring the first workers for each pipeline.
+
+<img class="center-block"
+src="/images/blog/apache-beam-flink-and-kubernetes-part2/flink-partition-assignment.png"
+alt="Flink split assignment on slots">
+
+To address this issue, we developed a custom patch for Flink's *SlotSharingSlotAllocator*, a component responsible for task distribution. This patch ensures a more balanced workload distribution across all available workers, improving efficiency and preventing bottlenecks.
+With this improvement, each worker gets a fair share of tasks, leading to better resource utilization and smoother operation of our Beam Jobs.
+
+## Drain support in Kubernetes Operator with Flink
+
+### The challenge
+
+In the world of data processing with Apache Flink, a common task is to manage and update data-processing jobs. These jobs could be either stateful, where they remember past data, or stateless, where they don't.
+
+In the past, when we needed to update or delete a Flink job managed by the Kubernetes Operator, the system saved the current state of the job using a savepoint or checkpoint. However, a crucial step was missing: the system didn't stop the job from processing new data (this is what we mean by draining the job). This oversight could lead to two major issues:
+1. **For stateful jobs:** potential data inconsistencies, because the job might process new data that wasn't accounted for in the savepoint
+2. **For stateless jobs:** data duplication, because the job might reprocess data it already processed
+
+### The solution: drain function
+
+This is where the update referenced as [FLINK-32700](https://issues.apache.org/jira/browse/FLINK-32700) is needed. This update introduced a drain function. Think of it as telling the job, "Finish what you're currently processing, but don't take on anything new." Here's how it works:
+1. **Stop new data:** The job stops reading new input.
+2. **Mark the source:** The job marks the source with an infinite watermark. Think of this watermark as a marker that tells the system that there's no more new data to process.
+3. **Propagate through the pipeline:** This marker is then passed through the job's processing pipeline, ensuring that every part of the job knows not to expect any new data.
+
+This seemingly small change has a big impact. It ensures that when a job is updated or deleted, the data it processes remains consistent and accurate. This is crucial for any data-processing task, because it maintains the integrity and reliability of the data. Furthermore, in cases where the drainage fails, you can cancel the job without needing a savepoint, which adds a layer of flexibility and safety to the whole process.
+
+## Conclusion
+
+As we conclude Part 2 of our series on building and managing Apache Beam Flink services on Kubernetes, it's evident that the journey of implementing autoscaling has been both challenging and enlightening. The obstacles we faced, from understanding Apache Beam backlog metrics in the Flink Runner environment to addressing slow reads in high-traffic scenarios, pushed us to develop innovative solutions and deepen our understanding of streaming infrastructure.
+
+Our exploration into the intricacies of checkpointing, Kafka Reader wait times, and unbalanced partition distribution revealed the complexities of autoscaling Beam jobs. These challenges prompted us to devise strategies like the adaptive timeout in `KafkaIO` and the balanced workload distribution in Flink's `SlotSharingSlotAllocator`. Additionally, the introduction of the drain support in Kubernetes Operator with Flink marks a significant advancement in managing stateful and stateless jobs effectively.
+
+This journey has not only enhanced the robustness and efficiency of our system but has also contributed valuable insights to the broader community working with Apache Beam and Flink. We hope that our experiences and solutions will aid others facing similar challenges in their projects.
+
+Stay tuned for our next blog post, where we'll delve into the specifics of autoscaling in Apache Beam. We'll break down the concepts, strategies, and best practices to effectively scale your Beam jobs. Thank you for following our series, and we look forward to sharing more of our journey and learnings with you.
+
+## Acknowledgements
+
+This is a large effort to build the new infrastructure and to migrate the large customer based applications from cloud provider managed streaming infrastructure to self-managed, Flink-based infrastructure at scale. Thanks the Palo Alto Networks CDL streaming team who helped to make this happen: Kishore Pola, Andrew Park, Hemant Kumar, Manan Mangal, Helen Jiang, Mandy Wang, Praveen Kumar Pasupuleti, JM Teo, Rishabh Kedia, Talat Uyarer, Naitk Dani, and David He.
+
+---
+
+**Explore More:**
+
+- [Part 1: Introduction to Building and Managing Apache Beam Flink Services on Kubernetes](https://beam.apache.org/blog/apache-beam-flink-and-kubernetes/)
+
+*Join the conversation and share your experiences on our [Community](https://beam.apache.org/community/) or contribute to our ongoing projects on [GitHub](https://github.com/apache/beam). Your feedback is invaluable. If you have any comments or questions about this series, please feel free to reach out to us via [User Mailist](https://beam.apache.org/community/contact-us/)*
+
+*Stay connected with us for more updates and insights into Apache Beam, Flink, and Kubernetes.*
diff --git a/website/www/site/content/en/blog/beam-2.52.0.md b/website/www/site/content/en/blog/beam-2.52.0.md
index 5654f16ceb37..2e604c8fabf8 100644
--- a/website/www/site/content/en/blog/beam-2.52.0.md
+++ b/website/www/site/content/en/blog/beam-2.52.0.md
@@ -41,7 +41,7 @@ should handle this. ([#25252](https://github.com/apache/beam/issues/25252)).
 * Add `UseDataStreamForBatch` pipeline option to the Flink runner. When it is set to true, Flink runner will run batch
   jobs using the DataStream API. By default the option is set to false, so the batch jobs are still executed
   using the DataSet API.
-* `upload_graph` as one of the Experiments options for DataflowRunner is no longer required when the graph is larger than 10MB for Java SDK ([PR#28621](https://github.com/apache/beam/pull/28621).
+* `upload_graph` as one of the Experiments options for DataflowRunner is no longer required when the graph is larger than 10MB for Java SDK ([PR#28621](https://github.com/apache/beam/pull/28621)).
 * state amd side input cache has been enabled to a default of 100 MB. Use `--max_cache_memory_usage_mb=X` to provide cache size for the user state API and side inputs. (Python) ([#28770](https://github.com/apache/beam/issues/28770)).
 * Beam YAML stable release. Beam pipelines can now be written using YAML and leverage the Beam YAML framework which includes a preliminary set of IO's and turnkey transforms. More information can be found in the YAML root folder and in the [README](https://github.com/apache/beam/blob/master/sdks/python/apache_beam/yaml/README.md).
 
@@ -69,69 +69,136 @@ as a workaround, a copy of "old" `CountingSource` class should be placed into a
 According to git shortlog, the following people contributed to the 2.52.0 release. Thank you to all contributors!
 
 Ahmed Abualsaud
+
 Ahmet Altay
+
 Aleksandr Dudko
+
 Alexey Romanenko
+
 Anand Inguva
+
 Andrei Gurau
+
 Andrey Devyatkin
+
 BjornPrime
+
 Bruno Volpato
+
 Bulat
+
 Chamikara Jayalath
+
 Damon
+
 Danny McCormick
+
 Devansh Modi
+
 Dominik Dębowczyk
+
 Ferran Fernández Garrido
+
 Hai Joey Tran
+
 Israel Herraiz
+
 Jack McCluskey
+
 Jan Lukavský
+
 JayajP
+
 Jeff Kinard
+
 Jeffrey Kinard
+
 Jiangjie Qin
+
 Jing
+
 Joar Wandborg
+
 Johanna Öjeling
+
 Julien Tournay
+
 Kanishk Karanawat
+
 Kenneth Knowles
+
 Kerry Donny-Clark
+
 Luís Bianchin
+
 Minbo Bae
+
 Pranav Bhandari
+
 Rebecca Szper
+
 Reuven Lax
+
 Ritesh Ghorse
+
 Robert Bradshaw
+
 Robert Burke
+
 RyuSA
+
 Shunping Huang
+
 Steven van Rossum
+
 Svetak Sundhar
+
 Tony Tang
+
 Vitaly Terentyev
+
 Vivek Sumanth
+
 Vlado Djerek
+
 Yi Hu
+
 aku019
+
 brucearctor
+
 caneff
+
 damccorm
+
 ddebowczyk92
+
 dependabot[bot]
+
 dpcollins-google
+
 edman124
+
 gabry.wu
+
 illoise
+
 johnjcasey
+
 jonathan-lemos
+
 kennknowles
+
 liferoad
+
 magicgoody
+
 martin trieu
+
 nancyxu123
+
 pablo rodriguez defino
+
 tvalentyn
+
diff --git a/website/www/site/content/en/blog/beam-2.53.0.md b/website/www/site/content/en/blog/beam-2.53.0.md
new file mode 100644
index 000000000000..14c55956bc13
--- /dev/null
+++ b/website/www/site/content/en/blog/beam-2.53.0.md
@@ -0,0 +1,185 @@
+---
+title:  "Apache Beam 2.53.0"
+date:   2024-01-04 09:00:00 -0400
+categories:
+  - blog
+  - release
+authors:
+  - jrmccluskey
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+We are happy to present the new 2.53.0 release of Beam.
+This release includes both improvements and new functionality.
+See the [download page](/get-started/downloads/) for this release.
+
+<!--more-->
+
+For more information on changes in 2.53.0, check out the [detailed release notes](https://github.com/apache/beam/milestone/17).
+
+## Highlights
+
+* Python streaming users that use 2.47.0 and newer versions of Beam should update to version 2.53.0, which fixes a known issue: ([#27330](https://github.com/apache/beam/issues/27330)).
+
+## I/Os
+
+* TextIO now supports skipping multiple header lines (Java) ([#17990](https://github.com/apache/beam/issues/17990)).
+* Python GCSIO is now implemented with GCP GCS Client instead of apitools ([#25676](https://github.com/apache/beam/issues/25676))
+* Adding support for LowCardinality DataType in ClickHouse (Java) ([#29533](https://github.com/apache/beam/pull/29533)).
+* Added support for handling bad records to KafkaIO (Java) ([#29546](https://github.com/apache/beam/pull/29546))
+* Add support for generating text embeddings in MLTransform for Vertex AI and Hugging Face Hub models.([#29564](https://github.com/apache/beam/pull/29564))
+* NATS IO connector added (Go) ([#29000](https://github.com/apache/beam/issues/29000)).
+
+## New Features / Improvements
+
+* The Python SDK now type checks `collections.abc.Collections` types properly. Some type hints that were erroneously allowed by the SDK may now fail. ([#29272](https://github.com/apache/beam/pull/29272))
+* Running multi-language pipelines locally no longer requires Docker.
+  Instead, the same (generally auto-started) subprocess used to perform the
+  expansion can also be used as the cross-language worker.
+* Framework for adding Error Handlers to composite transforms added in Java ([#29164](https://github.com/apache/beam/pull/29164)).
+* Python 3.11 images now include google-cloud-profiler ([#29561](https://github.com/apache/beam/pull/29651)).
+
+## Deprecations
+
+* Euphoria DSL is deprecated and will be removed in a future release (not before 2.56.0) ([#29451](https://github.com/apache/beam/issues/29451))
+
+## Bugfixes
+
+* (Python) Fixed sporadic crashes in streaming pipelines that affected some users of 2.47.0 and newer SDKs ([#27330](https://github.com/apache/beam/issues/27330)).
+* (Python) Fixed a bug that caused MLTransform to drop identical elements in the output PCollection ([#29600](https://github.com/apache/beam/issues/29600)).
+
+## Security Fixes
+
+* Upgraded to go 1.21.5 to build, fixing [CVE-2023-45285](https://security-tracker.debian.org/tracker/CVE-2023-45285) and [CVE-2023-39326](https://security-tracker.debian.org/tracker/CVE-2023-39326)
+
+## List of Contributors
+
+According to git shortlog, the following people contributed to the 2.53.0 release. Thank you to all contributors!
+
+Ahmed Abualsaud
+
+Ahmet Altay
+
+Alexey Romanenko
+
+Anand Inguva
+
+Arun Pandian
+
+Balázs Németh
+
+Bruno Volpato
+
+Byron Ellis
+
+Calvin Swenson Jr
+
+Chamikara Jayalath
+
+Clay Johnson
+
+Damon
+
+Danny McCormick
+
+Ferran Fernández Garrido
+
+Georgii Zemlianyi
+
+Israel Herraiz
+
+Jack McCluskey
+
+Jacob Tomlinson
+
+Jan Lukavský
+
+JayajP
+
+Jeffrey Kinard
+
+Johanna Öjeling
+
+Julian Braha
+
+Julien Tournay
+
+Kenneth Knowles
+
+Lawrence Qiu
+
+Mark Zitnik
+
+Mattie Fu
+
+Michel Davit
+
+Mike Williamson
+
+Naireen
+
+Naireen Hussain
+
+Niel Markwick
+
+Pablo Estrada
+
+Radosław Stankiewicz
+
+Rebecca Szper
+
+Reuven Lax
+
+Ritesh Ghorse
+
+Robert Bradshaw
+
+Robert Burke
+
+Sam Rohde
+
+Sam Whittle
+
+Shunping Huang
+
+Svetak Sundhar
+
+Talat UYARER
+
+Tom Stepp
+
+Tony Tang
+
+Vlado Djerek
+
+Yi Hu
+
+Zechen Jiang
+
+clmccart
+
+damccorm
+
+darshan-sj
+
+gabry.wu
+
+johnjcasey
+
+liferoad
+
+lrakla
+
+martin trieu
+
+tvalentyn
\ No newline at end of file
diff --git a/website/www/site/content/en/blog/scaling-streaming-workload.md b/website/www/site/content/en/blog/scaling-streaming-workload.md
new file mode 100644
index 000000000000..cbd642153f3c
--- /dev/null
+++ b/website/www/site/content/en/blog/scaling-streaming-workload.md
@@ -0,0 +1,291 @@
+---
+layout: post
+title:  "Scaling a streaming workload on Apache Beam, 1 million events per second and beyond"
+date:   2024-01-03 00:00:01 -0800
+categories:
+  - blog
+authors:
+  - pabs
+---
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Scaling a streaming workload on Apache Beam
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/0-intro.png"
+    alt="Streaming Processing">
+
+Scaling a streaming workload is critical for ensuring that a pipeline can process large amounts of data while also minimizing latency and executing efficiently. Without proper scaling, a pipeline may experience performance issues or even fail entirely, delaying the time to insights for the business.
+
+Given the Apache Beam support for the sources and sinks needed by the workload, developing a streaming pipeline can be easy. You can focus on the processing (transformations, enrichments, or aggregations) and on setting the right configurations for each case.
+
+However, you need to identify the key performance bottlenecks and make sure that the pipeline has the resources it needs to handle the load efficiently. This can involve right-sizing the number of workers, understanding the settings needed for the source and sinks of the pipeline, optimizing the processing logic, and even determining the transport formats.
+
+This article illustrates how to manage the problem of scaling and optimizing a streaming workload developed in Apache Beam and run on Google Cloud using Dataflow. The goal is to reach one million events per second, while also minimizing latency and resource use during execution. The workload uses Pub/Sub as the streaming source and BigQuery as the sink. We describe the reasoning behind the configuration settings and code changes we used to help the workload achieve the desired scale and beyond.
+
+The progression described in this article maps to the evolution of a real-life workload, with simplifications. After the initial business requirements for the pipeline were achieved, the focus shifted to optimizing the performance and reducing the resources needed for the pipeline execution.
+
+## Execution setup
+
+For this article, we created a test suite that creates the necessary components for the pipelines to execute. You can find the code in [this Github repository](https://github.com/prodriguezdefino/apache-beam-streaming-tests). You can find the subsequent configuration changes that are introduced on every run in this [folder](https://github.com/prodriguezdefino/apache-beam-streaming-tests/tree/main/scaling-streaming-workload-blog) as scripts that you can run to achieve similar results.
+
+All of the execution scripts can also execute a Terraform-based automation to create a Pub/Sub topic and subscription as well as a BigQuery dataset and table to run the workload. Also, it launches two pipelines: one data generation pipeline that pushes events to the Pub/Sub topic, and an ingestion pipeline that demonstrates the potential improvement points.
+
+In all cases, the pipelines start with an empty Pub/Sub topic and subscription and an empty BigQuery table. The plan is to generate one million events per second and, after a few minutes, review how the ingestion pipeline scales with time. The data being autogenerated is based on provided schemas or IDL (or Interface Description Language) given the configuration, and the goal is to have messages ranging between 800 bytes and 2 KB, adding up to approximately 1 GB/s volume throughput. Also, the ingestion pipelines are using the same worker type configuration on all runs (`n2d-standard-4` GCE machines) and are capping the maximum workers number to avoid very large fleets.
+
+All of the executions run on Google Cloud using Dataflow, but you can apply all of the configurations and format changes to the suite while executing on other supported Apache Beam runners. Changes and recommendations are not runner specific.
+
+### Local environment requirements
+
+Before launching the startup scripts, install the following items in your local environment:
+
+*   `gcloud`, along with the correct permissions
+*   Terraform
+*   JDK 17 or later
+*   Maven 3.6 or later
+
+For more information, see the [requirements](https://github.com/prodriguezdefino/apache-beam-streaming-tests#requisites) section in the GitHub repository.
+
+Also, review the service quotas and resources available in your Google Cloud project. Specifically: Pub/Sub regional capacity, BigQuery ingestion quota, and Compute Engine instances available in the selected region for the tests.
+
+### Workload description
+
+Focusing on the ingestion pipeline, our [workload](https://github.com/prodriguezdefino/apache-beam-streaming-tests/blob/main/canonical-streaming-pipelines/src/main/java/com/google/cloud/pso/beam/pipelines/StreamingSourceToBigQuery.java#L55) is straightforward. It completes the following steps:
+
+1. reads data in a specific format from Pub/Sub (Apache Thrift in this case)
+2. deals with potential compression and batching settings (not enabled by default)
+3. executes a UDF (identity function by default)
+4. transforms the input format to one of the formats supported by the `BigQueryIO` transform
+5. writes the data to the configured table
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/0-pipeline.png"
+    alt="Example Workload">
+
+The pipeline we used for the tests is highly configurable. For more details about how to tweak the ingestion, see the [options](https://github.com/prodriguezdefino/apache-beam-streaming-tests/blob/main/canonical-streaming-pipelines/src/main/java/com/google/cloud/pso/beam/pipelines/StreamingSourceToBigQuery.java#L39) in the file. No code changes are needed on any of our steps. The execution scripts take care of the configurations needed.
+
+Although these tests are focused on reading data from Pub/Sub, the ingestion pipeline is capable of reading data from a generic streaming source. The repository contains other [examples](https://github.com/prodriguezdefino/apache-beam-streaming-tests/tree/main/example-suite-scripts) that show how to launch this same test suite reading data from Pub/Sub Lite and Kafka. In all cases, the pipeline automation sets up the streaming infrastructure.
+
+Finally, you can see in the [configuration options](https://github.com/prodriguezdefino/apache-beam-ptransforms/blob/a0dd229081625c7b593512543614daf995a9f870/common/src/main/java/com/google/cloud/pso/beam/common/formats/options/TransportFormatOptions.java) that the pipeline supports many transport format options for the input, such as Thrift, Avro, and JSON. This suite focuses on Thrift, because it is a common open source format, and because it generates a format transformation need. The intent is to put some strain in the workload processing. You can run similar tests for Avro and JSON input data. The streaming data generator pipeline can generate random data for the [three supported formats](https://github.com/prodriguezdefino/apache-beam-streaming-tests/tree/main/streaming-data-generator/src/main/java/com/google/cloud/pso/beam/generator/formats) by walking directly on the schema (Avro and JSON) or IDL (Thrift) provided for execution.
+
+## First run: default settings
+
+The default values for the execution writes the data to BigQuery using `STREAMING_INSERTS` mode for `BigQueryIO`. This mode correlates with the [`tableData insertAll` API](https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll) for BigQuery. This API supports data in JSON format. From the Apache Beam perspective, using the `BigQueryIO.writeTableRows` method lets us resolve the writes into BigQuery.
+
+For our ingestion pipeline, the Thrift format needs to be transformed into `TableRow`. To do that, we need to translate the Thrift IDL into a BigQuery table schema. That can be achieved by translating the Thrift IDL into an Avro schema, and then using Beam utilities to translate the table schema for BigQuery. We can do this at bootstrap. The schema transformation is cached at the `DoFn` level.
+
+After setting up the data generation and ingestion pipelines, and after letting the pipelines run for some minutes, we see that the pipeline is unable to sustain the desired throughput.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/1-default-ps.png"
+    alt="PubSub metrics">
+
+The previous image shows that the number of messages that are not being processed by the ingestion pipeline start to show as unacknowledged messages in Pub/Sub metrics.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/1-default-throughput.png"
+    alt="Throughput">
+
+Reviewing the per stage performance metrics, we see that the pipeline shows a saw-like shape, which is often associated with the throttling mechanisms the Dataflow runner uses when some of the stages are acting as bottlenecks for the throughput. Also, we see that the `Reshuffle` step on the `BigQueryIO` write transform does not scale as expected.
+
+This behavior happens because by default the [`BigQueryOptions`](https://github.com/apache/beam/blob/master/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryOptions.java#L57) uses 50 different keys to shuffle data to workers before the writes happen on BigQuery. To solve this problem, we can add a configuration to our launch script that enables the write operations to scale to a larger number of workers, which improves performance.
+
+
+## Second run: improve the write bottleneck
+
+After increasing the number of streaming keys to a higher number, 512 keys in our case, we restarted the test suite. The Pub/Sub metrics started to improve. After an initial ramp on the size of the backlog, the curve started to ease out.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/2-skeys-ps.png"
+    alt="PubSub metrics">
+
+This is good, but we should take a look at the throughput per stage numbers to understand if we are achieving the goal we set up for this exercise.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/2-skeys-throughput.png"
+    alt="Throughput">
+
+Although the performance has clearly improved, and the Pub/Sub backlog no longer increases monotonically, we are still far from the goal of processing one million events per second (1 GB/s) for our ingestion pipeline. In fact, the throughput metrics jump all over, indicating that bottlenecks are preventing the processing from scaling further.
+
+## Third run: unleash autoscale
+
+Luckily for us, when writing into BigQuery, we can autoscale the writes. This step simplifies the configuration so that we don't have to guess the right number of shards. We switched the pipeline’s configuration and enabled this setting for the next [launch script](https://github.com/prodriguezdefino/apache-beam-streaming-tests/blob/main/scaling-streaming-workload-blog/3-ps2bq-si-tr-streamingautoshard.sh).
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/3-autoshard-parallelism.png"
+    alt="Key Parallelism">
+
+Immediately, we see that the autosharding mechanism tweaks the number of keys very aggressively and in a dynamic way. This change is good, because different moments in time might have different scale needs, such as early backlog recoveries and spikes in the execution.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/3-autoshard-throughput-tr.png"
+    alt="Throughput">
+
+Inspecting the throughput performance per stage, we see that as the number of keys increases, the performance of the writes also increases. In fact, it reaches very large numbers!
+
+After the initial backlog was consumed and the pipeline stabilized, we saw that the desired performance numbers were reached. The pipeline can sustain processing many more than a million events per second from Pub/Sub and several GB/s of BigQuery ingestion. Yay!
+
+Still, we want to see if we can do better. We can introduce several improvements to the pipeline to make the execution more efficient. In most cases, the improvements are configuration changes. We just need to know where to focus next.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/3-autoshard-autoscale.png"
+    alt="Resources">
+
+The previous image shows that the number of workers needed to sustain this throughput is still quite high. The workload itself is not CPU intensive. Most of the cost is spent on transforming formats and on I/O interactions, such as shuffles and the actual writes. To understand what to improve, we first investigate the transport formats.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/3-autoshard-tr-input.png"
+    alt="Thrift Input Size">
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/3-autoshard-tr-output.png"
+    alt="TableRow Output Size">
+
+Looking at the input size, right before the identity UDF execution, the data format is binary Thrift, which is a decently compact format even when no compression is used. However, while comparing the `PCollection` approximated size with the `TableRow` format needed for BigQuery ingestion, a clear size increase is visible. This is something we can improve by changing the BigQuery write API in use.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/3-autoshard-tr-overhead.png"
+    alt="Translation Overhead">
+
+When we inspect the `StoreInBigQuery` transform, we see that the majority of the wall time is spent on the actual writes. Also, the wall time spent converting data to the destination format (`TableRows`) compared with how much is spent in the actual writes is quite large: 13 times bigger for the writes. To improve this behavior, we can switch the pipeline write mode.
+
+## Fourth run: in with the new
+
+In this run, we use  the `StorageWrite` API. Enabling the `StorageWrite` API for this pipeline is straightforward. We set the write mode as `STORAGE_WRITE_API` and define a write triggering frequency. For this test, we write data at most every ten seconds. The write triggering frequency controls how long the per-stream data accumulate. A higher number defines a larger output to be written after the stream assignment but also imposes a larger end-to-end latency for every element read from Pub/Sub. Similar to the `STREAMING_WRITES` configuration, `BigQueryIO` can handle autosharding for the writes, which we already demonstrated to be the best setting for performance.
+
+After both pipelines become stable, the performance benefits seen when using the `StorageWrite` API in `BigQueryIO` are apparent. After enabling the new implementation, the wall time rate between the format transformation and write operation decreases. The wall time spent on writes is only about 34 percent larger than the format transformation.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/4-format-transformation.png"
+    alt="Translation Overhead">
+
+After stabilization, the pipeline throughput is also quite smooth. The runner can quickly and steadily downscale the pipeline resources needed to sustain the desired throughput.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/4-throughput.png"
+    alt="Throughput">
+
+Looking at the resource scale needed to process the data, another dramatic improvement is visible. Whereas the streaming inserts-based pipeline needed more than 80 workers to sustain the throughput, the storage writes pipeline only needs 49, a 40 percent improvement.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/4-ingestion-scale.png"
+    alt="Resources">
+
+We can use the data generation pipeline as reference. This pipeline only needs to randomly generate data and write the events to Pub/Sub. It runs steadily with an average of 40 workers. The improvements on the ingestion pipeline using the right configuration for the workload makes it closer to those resources needed for the generation.
+
+Similar to the streaming inserts-based pipeline, writing the data into BigQuery requires running a format translation, from Thrift to `TableRow` in the former and from Thrift to Protocol Buffers (protobuf) in the latter. Because we are using the `BigQueryIO.writeTableRows` method, we add another step in the format translation. Because the `TableRow` format also increases the size of the `PCollection` being processed, we want to see if we can improve this step.
+
+
+## Fifth run: a better write format
+
+When using `STORAGE_WRITE_API`, the `BigQueryIO` transform exposes a method that we can use to write the Beam row type directly into BigQuery. This step is useful because of the flexibility that the row type provides for interoperability and schema management. Also, it's both efficient for shuffling and denser than `TableRow`, so our pipeline will have smaller `PCollection` sizes.
+
+For the next run, because our data volume is not small, we decrease the triggering frequency when writing to BigQuery. Because we use a different format, slightly different code runs. For this change, the test pipeline script is configured with the flag `--formatToStore=BEAM_ROW`.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/5-input-size.png"
+    alt="Thrift input size">
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/5-output-size.png"
+    alt="Row output size">
+
+The `PCollection` size written into BigQuery is considerably smaller than on previous executions. In fact, for this particular execution, the Beam row format is a smaller size than the Thrift format. A larger `PCollection` conformed by bigger per-element sizes can put nontrivial memory pressure in smaller worker configurations, reducing the overall throughput.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/5-format-trasformation.png"
+    alt="Translation overhead">
+
+The wall clock rate for the format transformation and the actual BigQuery writes also maintain a very similar rate. Handling the Beam row format does not impose a performance penalty in the format translation and subsequent writes. This is confirmed by the number of workers in use by the pipeline when throughput becomes stable, slightly smaller than the previous run but clearly in the same range.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/5-ingestion-scale.png"
+    alt="Resources">
+
+Although we are in a much better position than when we started, given our test pipeline input format, there's still room for improvement.
+
+## Sixth run: further reduce the format translation effort
+
+Another supported format for the input `PCollection` in the `BigQueryIO` transform might be advantageous for our input format. The method `writeGenericRecords` enables the transform to transform Avro `GenericRecords` directly into protobuf before the write operation. Apache Thrift can be transformed into Avro `GenericRecords` very efficiently. We can make another test run configuring our test ingestion pipeline by setting the option `--formatToStore=AVRO_GENERIC_RECORD` on our execution script.
+
+This time, the difference between format translation and writes increases significantly, improving performance. The translation to Avro `GenericRecords` is only 20 percent of the write effort spent on writing those records into BigQuery. Given that the test pipelines had similar runtimes and that the wall clock seen in the `WriteIntoBigQuery` stage is also aligned with other `StorageWrite` related runs, using this format is appropriate for this workload.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/6-format-transformation.png"
+    alt="Translation overhead">
+
+We see further gains when we look at resource utilization. We need less CPU time to execute the format translations for our workload while achieving the desired throughput.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/6-ingestion-scale.png"
+    alt="Resources">
+
+This pipeline improves upon the previous run, running steadily on 42 workers when throughput is stable. Given the worker configuration used (`nd2-standard-4`), and the volume throughput of the workload process (about 1 GB/s), we are achieving about 6 MB/s throughput per CPU core, which is quite impressive for a streaming pipeline with exactly-once semantics.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/6-latencies.png"
+    alt="Latencies">
+
+When we add up all of the stages executed in the main path of the pipeline, the latency seen at this scale achieves sub-second end-to-end latencies during sustained periods of time.
+
+Given the workload requirements and the implemented pipeline code, this performance is the best that we can extract without further tuning the runner’s specific settings.
+
+## Seventh run : lets just relax (at least some constraints)
+
+When using the `STORAGE\_WRITE\_API` setting for `BigQueryIO`, we enforce exactly-once semantics on the writes. This configuration is great for use cases that need strong consistency on the data that gets processed, but it imposes a performance and cost penalty.
+
+From a high-level perspective, writes into BigQuery are made in batches, which are released based on the current sharding and the triggering frequency. If a write fails during the execution of a particular bundle, it is retried. A bundle of data is committed into BigQuery only when all the data in that particular bundle is correctly appended to a stream. This implementation needs to shuffle the full volume of data to create the batches that are written, and also the information of the finished batches for later commit (although this last piece is very small compared with the first).
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/7-previous-data-input.png"
+    alt="Read data size">
+
+Looking at the previous pipeline execution, the total data being processed for the pipeline by Streaming Engine is larger than the data being read from Pub/Sub. For example, 7 TB of data is read from Pub/Sub, whereas the processing of data for the whole execution of the pipeline moves 25 TB of data to and from Streaming Engine.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/7-previous-shuffle-total.png"
+    alt="Streamed data size">
+
+When data consistency is not a hard requirement for ingestion, you can use at-least-once semantics with `BigQueryIO` write mode. This implementation avoids shuffling and grouping data for the writes. However, this change might cause a small number of repeated rows to be written into the destination table. This can happen with append errors, infrequent worker restarts, and other even less frequent errors.
+
+Therefore, we add the configuration to use `STORAGE_API_AT_LEAST_ONCE` write mode. To instruct the `StorageWrite` client to reuse connections while writing data, we also add the configuration flag `–useStorageApiConnectionPool`. This configuration option only works with `STORAGE_API_AT_LEAST_ONCE` mode, and it reduces the occurrences of warnings similars to `Storage Api write delay more than 8 seconds`.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/7-resources.png"
+    alt="Resources">
+
+When pipeline throughput stabilizes, we see a similar pattern for resource utilization for the workload. The number of workers in use reaches 40, a small improvement compared with the last run. However, the amount of data being moved from Streaming Engine is much closer to the amount of data read from Pub/Sub.
+
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/7-current-input.png"
+    alt="Read data size">
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/7-current-shuffle-total.png"
+    alt="Streamed data size">
+
+Considering all of these factors, this change further optimizes the workload, achieving a throughput of 6.4 MB/s per CPU core. This improvement is small compared to the same workload when using consistent writes into BigQuery, but it uses less streaming data resources. This configuration represents the most optimal setup for our workload, with the highest throughput per resource and the lowest streaming data across workers.
+
+<img class="center-block"
+    src="/images/blog/scaling-streaming-workload/7-latency.png"
+    alt="Streamed data size">
+
+This configuration also has impressively low latency for the end-to-end processing. Given that the main path of our pipeline has been fused in a single execution stage from reads to writes, we see that even at p99, the latency tends to be below 300 milliseconds at a quite large volume throughput (as previously mentioned around 1 GB/s).
+
+## Recap
+
+Optimizing Apache Beam streaming workloads for low latency and efficient execution requires careful analysis and decision-making, and the right configurations.
+
+Considering the scenario discussed in this article, it is essential to consider factors like overall CPU utilization, throughput and latency per stage, `PCollection` sizes, wall time per stage, write mode, and transport formats, in addition to writing the right pipeline for the workload.
+
+Our experiments revealed that using the `StorageWrite` API, autosharding for writes, and Avro `GenericRecords` as the transport format yielded the most efficient results. Relaxing the consistency for writes can further improve performance.
+
+The accompanying [Github repository](https://github.com/prodriguezdefino/apache-beam-streaming-tests) contains a test suite that you can use to replicate the analysis on your Google Cloud project or with a different runner setup. Feel free to take it for a spin. Comments and PRs are always welcome.
diff --git a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md
index 769b05741345..64d82d8ab959 100644
--- a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md
+++ b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md
@@ -502,6 +502,8 @@ fail later when the write attempts happen.
 
 If your BigQuery write operation creates a new table, you must provide schema
 information. The schema contains information about each field in the table.
+When updating a pipeline with a new schema, the existing schema fields must
+stay in the same order, or the pipeline will break, failing to write to BigQuery.
 
 {{< paragraph class="language-java" >}}
 To create a table schema in Java, you can either use a `TableSchema` object, or
diff --git a/website/www/site/content/en/documentation/ml/about-ml.md b/website/www/site/content/en/documentation/ml/about-ml.md
index 1753c0ed56c0..70ad546c40a1 100644
--- a/website/www/site/content/en/documentation/ml/about-ml.md
+++ b/website/www/site/content/en/documentation/ml/about-ml.md
@@ -61,7 +61,7 @@ To keep your model up to date and performing well as your data grows and evolves
 
 ## Use RunInference
 
-The [RunInference API](/documentation/sdks/python-machine-learning/) is a `PTransform` optimized for machine learning inferences that lets you efficiently use ML models in your pipelines. The API includes the following features:
+The RunInference API is a `PTransform` optimized for machine learning inferences that lets you efficiently use ML models in your pipelines. The API includes the following features:
 
 - To efficiently feed your model, dynamically batches inputs based on pipeline throughput using Apache Beam's `BatchElements` transform.
 - To balance memory and throughput usage, determines the optimal number of models to load using a central model manager. Shares these models across threads and processes as needed to maximize throughput.
diff --git a/website/www/site/content/en/documentation/ml/inference-overview.md b/website/www/site/content/en/documentation/ml/inference-overview.md
index a79c9a6b01e5..1821102c17fe 100644
--- a/website/www/site/content/en/documentation/ml/inference-overview.md
+++ b/website/www/site/content/en/documentation/ml/inference-overview.md
@@ -38,7 +38,7 @@ Beam provides different ways to implement inference as part of your pipeline. Yo
   </tr>
 </table>
 
-The RunInference API is available with the Beam Python SDK versions 2.40.0 and later. You can use Apache Beam with the RunInference API to use machine learning (ML) models to do local and remote inference with batch and streaming pipelines. Starting with Apache Beam 2.40.0, PyTorch and Scikit-learn frameworks are supported. Tensorflow models are supported through `tfx-bsl`. For more deatils about using RunInference with Python, see [Machine Learning with Python](/documentation/sdks/python-machine-learning/).
+The RunInference API is available with the Beam Python SDK versions 2.40.0 and later. You can use Apache Beam with the RunInference API to use machine learning (ML) models to do local and remote inference with batch and streaming pipelines. Starting with Apache Beam 2.40.0, PyTorch and Scikit-learn frameworks are supported. Tensorflow models are supported through `tfx-bsl`. For more deatils about using RunInference with Python, see [About Beam ML](/documentation/ml/about-ml).
 
 The RunInference API is available with the Beam Java SDK versions 2.41.0 and later through Apache Beam's [Multi-language Pipelines framework](/documentation/programming-guide/#multi-language-pipelines). For information about the Java wrapper transform, see [RunInference.java](https://github.com/apache/beam/blob/master/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/transforms/RunInference.java). To try it out, see the [Java Sklearn Mnist Classification example](https://github.com/apache/beam/tree/master/examples/multi-language).
 
@@ -47,7 +47,7 @@ You can create multiple types of transforms using the RunInference API: the API
 {{< table >}}
 | Task | Example |
 | ------- | ---------------|
-| I want to use the RunInference transform | [Modify a Python pipeline to use an ML model](/documentation/sdks/python-machine-learning/#modify-a-python-pipeline-to-use-an-ml-model) |
+| I want to use the RunInference transform | [Modify a Python pipeline to use an ML model](/documentation/ml/about-ml/#modify-a-python-pipeline-to-use-an-ml-model) |
 | I want to use RunInference with PyTorch | [Use RunInference with PyTorch](/documentation/transforms/python/elementwise/runinference-pytorch/) |
 | I want to use RunInference with Sklearn | [Use RunInference with Sklearn](/documentation/transforms/python/elementwise/runinference-sklearn/) |
 | I want to use pre-trained models (PyTorch, Scikit-learn, or TensorFlow) | [Use pre-trained models](/documentation/ml/about-ml/#use-pre-trained-models) |:
diff --git a/website/www/site/content/en/documentation/ml/multi-model-pipelines.md b/website/www/site/content/en/documentation/ml/multi-model-pipelines.md
index c42c8b8ae661..dcb96e17a468 100644
--- a/website/www/site/content/en/documentation/ml/multi-model-pipelines.md
+++ b/website/www/site/content/en/documentation/ml/multi-model-pipelines.md
@@ -33,7 +33,7 @@ all of those steps together by encapsulating them in a single Apache Beam Direct
 resilient and scalable end-to-end machine learning systems.
 
 To deploy your machine learning model in an Apache Beam pipeline, use
-the [`RunInferenceAPI`](/documentation/sdks/python-machine-learning/), which
+the [`RunInferenceAPI`](/documentation/ml/about-ml), which
 facilitates the integration of your model as a `PTransform` step in your DAG. Composing
 multiple `RunInference` transforms within a single DAG makes it possible to build a pipeline that consists
 of multiple ML models. In this way, Apache Beam supports the development of complex ML systems.
diff --git a/website/www/site/content/en/documentation/ml/overview.md b/website/www/site/content/en/documentation/ml/overview.md
index 63aaf0f86e20..3a49d40548e8 100644
--- a/website/www/site/content/en/documentation/ml/overview.md
+++ b/website/www/site/content/en/documentation/ml/overview.md
@@ -54,7 +54,7 @@ Beam provides different ways to implement inference as part of your pipeline. Yo
 
 ### RunInference
 
-The RunInfernce API is available with the Beam Python SDK versions 2.40.0 and later. You can use Apache Beam with the RunInference API to use machine learning (ML) models to do local and remote inference with batch and streaming pipelines. Starting with Apache Beam 2.40.0, PyTorch and Scikit-learn frameworks are supported. Tensorflow models are supported through `tfx-bsl`. For more deatils about using RunInference with Python, see [Machine Learning with Python](/documentation/sdks/python-machine-learning/).
+The RunInfernce API is available with the Beam Python SDK versions 2.40.0 and later. You can use Apache Beam with the RunInference API to use machine learning (ML) models to do local and remote inference with batch and streaming pipelines. Starting with Apache Beam 2.40.0, PyTorch and Scikit-learn frameworks are supported. Tensorflow models are supported through `tfx-bsl`. For more deatils about using RunInference, see [About Beam ML](/documentation/ml/about-ml).
 
 The RunInference API is available with the Beam Java SDK versions 2.41.0 and later through Apache Beam's [Multi-language Pipelines framework](/documentation/programming-guide/#multi-language-pipelines). For information about the Java wrapper transform, see [RunInference.java](https://github.com/apache/beam/blob/master/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/transforms/RunInference.java). To try it out, see the [Java Sklearn Mnist Classification example](https://github.com/apache/beam/tree/master/examples/multi-language).
 
@@ -63,7 +63,7 @@ You can create multiple types of transforms using the RunInference API: the API
 {{< table >}}
 | Task | Example |
 | ------- | ---------------|
-| I want to use the RunInference transform | [Modify a Python pipeline to use an ML model](/documentation/sdks/python-machine-learning/#modify-a-python-pipeline-to-use-an-ml-model) |
+| I want to use the RunInference transform | [Modify a Python pipeline to use an ML model](/documentation/ml/about-ml/#modify-a-python-pipeline-to-use-an-ml-model) |
 | I want to use RunInference with PyTorch | [Use RunInference with PyTorch](/documentation/transforms/python/elementwise/runinference-pytorch/) |
 | I want to use RunInference with Sklearn | [Use RunInference with Sklearn](/documentation/transforms/python/elementwise/runinference-sklearn/) |
 | I want to use pre-trained models (PyTorch, Scikit-learn, or TensorFlow) | [Use pre-trained models](/documentation/ml/about-ml/#use-pre-trained-models) |
diff --git a/website/www/site/content/en/documentation/ml/preprocess-data.md b/website/www/site/content/en/documentation/ml/preprocess-data.md
index 2b291b9c75a5..1365926d3cc1 100644
--- a/website/www/site/content/en/documentation/ml/preprocess-data.md
+++ b/website/www/site/content/en/documentation/ml/preprocess-data.md
@@ -23,16 +23,11 @@ preprocessing data for training and inference. The `MLTransform` class wraps the
 various transforms in one class, simplifying your workflow. For a full list of
 available transforms, see the [Transforms](#transforms) section on this page.
 
-The set of transforms currently available in the `MLTransform` class come from
-the TensorFlow Transforms (TFT) library. TFT offers specialized processing
-modules for machine learning tasks.
-
 ## Why use MLTransform {#use-mltransform}
 
 -   With `MLTransform`, you can use the same preprocessing steps for both
     training and inference, which ensures consistent results.
--   Use `MLTransform` to transform a single example or a batch of
-    examples.
+-   Generate [embeddings](https://en.wikipedia.org/wiki/Embedding) on text data using large language models (LLMs).
 -   `MLTransform` can do a full pass on the dataset, which is useful when
     you need to transform a single element only after analyzing the entire
     dataset. For example, with `MLTransform`, you can complete the following tasks:
@@ -45,18 +40,33 @@ modules for machine learning tasks.
     -   Count the occurrences of words in all the documents to calculate
         [TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf)
         weights.
-
 ## Support and limitations {#support}
 
--   Available in the Apache Beam Python SDK versions 2.50.0 and later.
--   Supports Python 3.8 and 3.9.
+-   Available in the Apache Beam Python SDK versions 2.53.0 and later.
+-   Supports Python 3.8, 3.9, and 3.10.
 -   Only available for pipelines that use [default windows](/documentation/programming-guide/#single-global-window).
--   Only supports one-to-one transform mapping on a single element.
 
 ## Transforms {#transforms}
 
-You can use `MLTransform` to perform the following data processing transforms.
-For information about the transforms, see
+You can use `MLTransform` to generate text embeddings and to perform various data processing transforms.
+
+### Text embedding transforms
+
+You can use `MLTranform` to generate embeddings that you can use to push data into vector databases or to run inference.
+
+{{< table >}}
+| Transform name | Description |
+| ------- | ---------------|
+| SentenceTransformerEmbeddings | Uses the Hugging Face [`sentence-transformers`](https://huggingface.co/sentence-transformers) models to generate text embeddings.
+| VertexAITextEmbeddings | Uses models from the [the Vertex AI text-embeddings API](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings) to generate text embeddings.
+{{< /table >}}
+
+
+### Data processing transforms that use TFT
+
+The following set of transforms available in the `MLTransform` class come from
+the TensorFlow Transforms (TFT) library. TFT offers specialized processing
+modules for machine learning tasks. For information about these transforms, see
 [Module:tft](https://www.tensorflow.org/tfx/transform/api_docs/python/tft) in the
 TensorFlow documentation.
 
@@ -73,18 +83,10 @@ TensorFlow documentation.
 | TFIDF | See [`tft.tfidf`](https://www.tensorflow.org/tfx/transform/api_docs/python/tft/tfidf) in the TensorFlow documentation. |:
 {{< /table >}}
 
-Apply the transforms on either single or multiple columns passed as a
-`dict` on structured data. Keys are column names and values are lists containing
-each column's data.
-
 ## I/O requirements {#io}
 
--   Input to the `MLTransform` class must be in one of the following formats:
-    -   A `dict` of `str`
-        -   Primitive types
-        -   List of primitive types
-        -   NumPy arrays
--   `MLTransform` outputs a Beam `Row` object with NumPy arrays.
+-   Input to the `MLTransform` class must be a dictionary.
+-   `MLTransform` outputs a Beam `Row` object with transformed elements.
 -   The output `PCollection` is a schema `PCollection`. The output schema
     contains the transformed columns.
 
@@ -197,32 +199,3 @@ Replace the following values:
 For more examples, see
 [MLTransform for data processing](/documentation/transforms/python/elementwise/mltransform)
 in the [transform catalog](/documentation/transforms/python/overview/).
-
-### ScaleTo01 example {#scaleto01}
-
-This example demonstrates how to use `MLTransform` to normalize your data
-between 0 and 1 by using the minimum and maximum values from your entire
-dataset. `MLTransform` uses the `ScaleTo01` transformation.
-
-Use the following snippet to apply `ScaleTo01` on column `x` of the input
-data.
-
-```
-data_pcoll | MLTransform(write_artifact_location=<LOCATION>).with_transform(ScaleTo01(columns=['x']))
-```
-
-The `ScaleTo01` transformation produces two artifacts: the `min` and the `max`
-of the entire dataset. For more information, see the
-[Artifacts](#artifacts) section on this page.
-
-## Metrics {#metrics}
-
-When you use MLTransform, the following metrics are available.
-
-{{< table >}}
-| Metric | Description |
-| ------- | ---------------|
-| Data throughput | The number of records processed per second. This metric indicates the processing capacity of the pipeline for `beam.MLTransform.` |
-| Memory usage | The number of records processed per second. This metric indicates the processing capacity of the pipeline for `beam.MLTransform`. |
-| Counters | Tracks the number of elements processed. Each `MLTransform` has a counter. |:
-{{< /table >}}
\ No newline at end of file
diff --git a/website/www/site/content/en/documentation/programming-guide.md b/website/www/site/content/en/documentation/programming-guide.md
index 92163c12ab8b..05a3232aa186 100644
--- a/website/www/site/content/en/documentation/programming-guide.md
+++ b/website/www/site/content/en/documentation/programming-guide.md
@@ -8105,40 +8105,64 @@ class RetrieveTimingDoFn(beam.DoFn):
 ## 15 Transform service {#transform-service}
 
 The Apache Beam SDK versions 2.49.0 and later include a [Docker Compose](https://docs.docker.com/compose/)
-service named _Transform service_. Use the Transform service to perform expansions of supported transforms
-on Beam portable pipelines by using Docker.
+service named _Transform service_.
 
 The following diagram illustrates the basic architecture of the Transform service.
 
 ![Diagram of the Transform service architecture](/images/transform_service.png)
 
-To use the Transform service, Docker and Docker Compose must be available on the machine that starts the service.
+To use the Transform service, Docker must be available on the machine that starts the service.
 
-The Transform service has the following primary use cases:
+The Transform service has several primary use cases.
 
-* Perform expansion of cross-language transforms without installing other language runtimes.
+### 15.1 Using the transform service to upgrade transforms {#transform-service-usage-upgrade}
 
-  The Transform service allows multi-language pipelines to use and expand cross-language transforms implemented
-  in other SDKs without requiring you to install runtimes for the implementation languages of those SDKs.
-  For example, with the Transform service, a Beam Python pipeline can use the Google Cloud Java I/O transforms and Java Kafka I/O transforms
-  without a local Java runtime installation.
+Transform service can be used to upgrade (or downgrade) the Beam SDK versions of supported individual transforms used by Beam pipelines without changing the Beam version of the pipelines.
+This feature is currently only available for Beam Java SDK 2.53.0 and later. Currently, the following transforms are available for upgrading:
 
-* Upgrade transforms without upgrading the Apache Beam SDK version.
+* BigQuery read transform (URN: _beam:transform:org.apache.beam:bigquery_read:v1_)
+* BigQuery write transform (URN: _beam:transform:org.apache.beam:bigquery_write:v1_)
+* Kafka read transform (URN: _beam:transform:org.apache.beam:kafka_read_with_metadata:v2_)
+* Kafka write transform (URN: _beam:transform:org.apache.beam:kafka_write:v2_)
 
-  Use the Transform service to upgrade the Beam SDK versions of individual transforms used by Beam pipelines without upgrading the Beam version of the pipeline.
-  This feature is currently in development. For more details, see
-  [GitHub issue #27943: Upgrade transforms without upgrading the pipeline using the Transform Service](https://github.com/apache/beam/issues/27943).
+To use this feature, you can simply execute a Java pipeline with additional pipeline options that specify the URNs of the transforms you would like to upgrade and the Beam version you would like to upgrade the transforms to. All transforms in the pipeline with matching URNs will be upgraded.
 
-### 15.1 Use the Transform service {#transform-service-usage}
+For example, to upgrade the BigQuery read transform for a pipeline run using Beam `2.53.0` to a future Beam version `2.xy.z`, you can specify the following additional pipelines options.
 
-In some cases, Apache Beam SDKs automatically start the Transform service, such as in the following scenarios:
+{{< highlight java >}}
+--transformsToOverride=beam:transform:org.apache.beam:bigquery_read:v1 --transformServiceBeamVersion=2.xy.z
+{{< /highlight >}}
+
+{{< highlight py >}}
+This feature is currently not available for Python SDK.
+{{< /highlight >}}
+
+{{< highlight go >}}
+This feature is currently not available for Go SDK.
+{{< /highlight >}}
+
+Note that the framework will automatically download the relevant Docker containers and startup the transform service for you.
+
+Please see [here](https://cwiki.apache.org/confluence/display/BEAM/Transform+Service#TransformService-Upgradetransformswithoutupgradingthepipeline) for a full example that uses this feature to upgrade BigQuery read and write transforms.
+
+### 15.2 Using the Transform service for multi-language pipelines {#transform-service-usage-multi-language}
+
+Transform service implements the Beam expansion API. This allows Beam multi-language pipelines to use the transform service when expanding transforms available within the transform service.
+The main advantage here is that multi-language pipelines will be able to operate without installing support for additional language runtimes. For example, Beam Python pipelines that use Java transforms such as
+`KafkaIO` can operate without installing Java locally during job submission as long as Docker is available in the system.
+
+In some cases, Apache Beam SDKs can automatically start the Transform service.
 
 * The Java [`PythonExternalTransform` API](https://github.com/apache/beam/blob/master/sdks/java/extensions/python/src/main/java/org/apache/beam/sdk/extensions/python/PythonExternalTransform.java) automatically
 starts the Transform service when a Python runtime isn't available locally, but Docker is.
 
 * The Apache Beam Python multi-language wrappers might automatically start the Transform service when you're using Java transforms, a Java language runtime isn't available locally, and Docker is available locally.
 
-To manually start a Transform service instance by using utilities provided with the Apache Beam SDKs, use the following commands.
+Beam users also have the option to [manually start](/documentation/programming-guide/#transform-service-usage-muanual) a transform service and use that as the expansion service used by multi-language pipelines.
+
+### 15.3 Manually starting the transform service {#transform-service-usage-muanual}
+
+A Beam Transform service instance can be manually started by using utilities provided with Apache Beam SDKs.
 
 {{< highlight java >}}
 java -jar beam-sdks-java-transform-service-launcher-<Beam version for the jar>.jar --port <port> --beam_version <Beam version for the transform service> --project_name <a unique ID for the transform service> --command up
@@ -8166,11 +8190,11 @@ python -m apache_beam.utils.transform_service_launcher --port <port> --beam_vers
 This feature is currently in development.
 {{< /highlight >}}
 
-### 15.2 Portable transforms included in the Transform service {#transform-service-included-transforms}
+### 15.4 Portable transforms included in the Transform service {#transform-service-included-transforms}
 
-The Transform service includes portable transforms implemented in the Apache Beam Java and Python SDKs.
+Beam Transform service includes a number of transforms implemented in the Apache Beam Java and Python SDKs.
 
-The following transforms are included in the Trasnform service:
+Currently, the following transforms are included in the Transform service:
 
 * Java transforms: Google Cloud I/O connectors, the Kafka I/O connector, and the JDBC I/O connector
 
@@ -8179,4 +8203,4 @@ The following transforms are included in the Trasnform service:
   [DataFrame](/documentation/dsls/dataframes/overview/) transforms.
 
 For a more comprehensive list of available transforms, see the
-[Transform service](https://cwiki.apache.org/confluence/display/BEAM/Transform+Service) developer guide.
+[Transform service](https://cwiki.apache.org/confluence/display/BEAM/Transform+Service#TransformService-TransformsincludedintheTransformservice) developer guide.
diff --git a/website/www/site/content/en/documentation/runners/direct.md b/website/www/site/content/en/documentation/runners/direct.md
index 26e23b4bd09f..85329d41d309 100644
--- a/website/www/site/content/en/documentation/runners/direct.md
+++ b/website/www/site/content/en/documentation/runners/direct.md
@@ -76,6 +76,10 @@ Local execution is limited by the memory available in your local environment. It
 
 ### Streaming execution
 
+{{< paragraph class="language-py" >}}
+Streaming support for Python DirectRunner is limited. For known issues, see: https://github.com/apache/beam/issues/24528.
+{{< /paragraph >}}
+
 If your pipeline uses an unbounded data source or sink, you must set the `streaming` option to `true`.
 
 ### Parallel execution
diff --git a/website/www/site/content/en/documentation/runtime/environments.md b/website/www/site/content/en/documentation/runtime/environments.md
index 452fb6141e63..3d730700e1f3 100644
--- a/website/www/site/content/en/documentation/runtime/environments.md
+++ b/website/www/site/content/en/documentation/runtime/environments.md
@@ -180,13 +180,13 @@ Beam offers a way to provide your own custom container image. The easiest way to
 1. Copy necessary artifacts from Apache Beam base image to your image.
   ```
   # This can be any container image,
- FROM python:3.7-bullseye
+ FROM python:3.8-bookworm
 
  # Install SDK. (needed for Python SDK)
- RUN pip install --no-cache-dir apache-beam[gcp]==2.35.0
+ RUN pip install --no-cache-dir apache-beam[gcp]==2.52.0
 
  # Copy files from official SDK image, including script/dependencies.
- COPY --from=apache/beam_python3.7_sdk:2.35.0 /opt/apache/beam /opt/apache/beam
+ COPY --from=apache/beam_python3.8_sdk:2.52.0 /opt/apache/beam /opt/apache/beam
 
  # Perform any additional customizations if desired
 
@@ -194,7 +194,7 @@ Beam offers a way to provide your own custom container image. The easiest way to
  ENTRYPOINT ["/opt/apache/beam/boot"]
 
   ```
->**NOTE**: This example assumes necessary dependencies (in this case, Python 3.7 and pip) have been installed on the existing base image. Installing the Apache Beam SDK into the image will ensure that the image has the necessary SDK dependencies and reduce the worker startup time.
+>**NOTE**: This example assumes necessary dependencies (in this case, Python 3.8 and pip) have been installed on the existing base image. Installing the Apache Beam SDK into the image will ensure that the image has the necessary SDK dependencies and reduce the worker startup time.
 >The version specified in the `RUN` instruction must match the version used to launch the pipeline.<br>
 >**Make sure that the Python or Java runtime version specified in the base image is the same as the version used to run the pipeline.**
 
@@ -202,7 +202,7 @@ Beam offers a way to provide your own custom container image. The easiest way to
 
 2. [Build](https://docs.docker.com/engine/reference/commandline/build/) and [push](https://docs.docker.com/engine/reference/commandline/push/) the image using Docker.
   ```
-    export BASE_IMAGE="apache/beam_python3.7_sdk:2.25.0"
+    export BASE_IMAGE="apache/beam_python3.8_sdk:2.52.0"
     export IMAGE_NAME="myremoterepo/mybeamsdk"
     export TAG="latest"
 
diff --git a/website/www/site/content/en/documentation/sdks/python.md b/website/www/site/content/en/documentation/sdks/python.md
index dc9f6a54d893..f51218327676 100644
--- a/website/www/site/content/en/documentation/sdks/python.md
+++ b/website/www/site/content/en/documentation/sdks/python.md
@@ -28,7 +28,7 @@ See the [Python API reference](https://beam.apache.org/releases/pydoc/) for more
 ## Python streaming pipelines
 
 Python [streaming pipeline execution](/documentation/sdks/python-streaming)
-is experimentally available (with some [limitations](/documentation/sdks/python-streaming/#unsupported-features))
+is available (with some [limitations](/documentation/sdks/python-streaming/#unsupported-features))
 starting with Beam SDK version 2.5.0.
 
 ## Python type safety
@@ -52,7 +52,7 @@ To integrate machine learning models into your pipelines for making inferences,
 [library from `tfx_bsl`](https://github.com/tensorflow/tfx-bsl/tree/master/tfx_bsl/beam).
 
 You can create multiple types of transforms using the RunInference API: the API takes multiple types of setup parameters from model handlers, and the parameter type determines the model implementation. For more information,
-see [Machine Learning](/documentation/sdks/python-machine-learning).
+see [About Beam ML](/documentation/ml/about-ml).
 
 [TensorFlow Extended (TFX)](https://www.tensorflow.org/tfx) is an end-to-end platform for deploying production ML pipelines. TFX is integrated with Beam. For more information, see [TFX user guide](https://www.tensorflow.org/tfx/guide).
 
diff --git a/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md b/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md
index 369b74714424..47944b9a232f 100644
--- a/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md
+++ b/website/www/site/content/en/documentation/transforms/python/elementwise/runinference.md
@@ -31,7 +31,7 @@ limitations under the License.
 
 Uses models to do local and remote inference. A `RunInference` transform performs inference on a `PCollection` of examples using a machine learning (ML) model. The transform outputs a `PCollection` that contains the input examples and output predictions. Avaliable in Apache Beam 2.40.0 and later versions.
 
-For more information, read about Beam RunInference APIs on [Machine Learning with Python](https://beam.apache.org/documentation/sdks/python-machine-learning) page and explore [RunInference API pipeline examples](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/inference).
+For more information about Beam RunInference APIs, see the [About Beam ML](https://beam.apache.org/documentation/ml/about-ml) page and the [RunInference API pipeline](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/inference) examples.
 
 ## Examples
 
diff --git a/website/www/site/content/en/get-started/_index.md b/website/www/site/content/en/get-started/_index.md
index 8aa6ff626c42..e6dfbc8ef2a3 100644
--- a/website/www/site/content/en/get-started/_index.md
+++ b/website/www/site/content/en/get-started/_index.md
@@ -21,7 +21,9 @@ limitations under the License.
 
 # Get Started with Apache Beam
 
-Learn how to use Beam to create data processing pipelines that run on supported processing back-ends.
+Learn how to use Beam to create data processing pipelines that run on supported processing back-ends. More learning resources are [here](https://beam.apache.org/get-started/resources/learning-resources/).
+
+You can also get a Beam badge now if you pass [this quest](https://beam.apache.org/get-started/resources/learning-resources/#certifications)!
 
 ## Tour of Beam
 
@@ -29,6 +31,10 @@ Learn how to use Beam to create data processing pipelines that run on supported
 Topics include core Beam concepts, from simple to advanced.
 You can try examples, do exercises, and solve challenges along the learning journey.
 
+## Beam Playground
+
+[Beam Playground](https://play.beam.apache.org) is an interactive environment to try out Beam transforms and examples without having to install Apache Beam in your environment.
+
 ## Beam Overview
 
 Read the [Apache Beam Overview](/get-started/beam-overview) to learn about the Beam model,
diff --git a/website/www/site/content/en/get-started/downloads.md b/website/www/site/content/en/get-started/downloads.md
index e426fa48f0d8..cc65d6d1f99f 100644
--- a/website/www/site/content/en/get-started/downloads.md
+++ b/website/www/site/content/en/get-started/downloads.md
@@ -96,10 +96,18 @@ versions denoted `0.x.y`.
 
 ## Releases
 
+### 2.53.0 (2024-01-04)
+Official [source code download](https://downloads.apache.org/beam/2.53.0/apache-beam-2.53.0-source-release.zip).
+[SHA-512](https://downloads.apache.org/beam/2.53.0/apache-beam-2.53.0-source-release.zip.sha512).
+[signature](https://downloads.apache.org/beam/2.53.0/apache-beam-2.53.0-source-release.zip.asc).
+
+[Release notes](https://github.com/apache/beam/releases/tag/v2.53.0)
+[Blog post](/blog/beam-2.53.0).
+
 ### 2.52.0 (2023-11-17)
-Official [source code download](https://downloads.apache.org/beam/2.52.0/apache-beam-2.52.0-source-release.zip).
-[SHA-512](https://downloads.apache.org/beam/2.52.0/apache-beam-2.52.0-source-release.zip.sha512).
-[signature](https://downloads.apache.org/beam/2.52.0/apache-beam-2.52.0-source-release.zip.asc).
+Official [source code download](https://archive.apache.org/beam/2.52.0/apache-beam-2.52.0-source-release.zip).
+[SHA-512](https://archive.apache.org/beam/2.52.0/apache-beam-2.52.0-source-release.zip.sha512).
+[signature](https://archive.apache.org/beam/2.52.0/apache-beam-2.52.0-source-release.zip.asc).
 
 [Release notes](https://github.com/apache/beam/releases/tag/v2.52.0)
 [Blog post](/blog/beam-2.52.0).
diff --git a/website/www/site/content/en/get-started/resources/learning-resources.md b/website/www/site/content/en/get-started/resources/learning-resources.md
index abd4566f4d3c..b983071401bb 100644
--- a/website/www/site/content/en/get-started/resources/learning-resources.md
+++ b/website/www/site/content/en/get-started/resources/learning-resources.md
@@ -69,7 +69,7 @@ If you have additional material that you would like to see here, please let us k
 
 ### Machine Learning
 
-*   **[Machine Learning with Python using the RunInference API](/documentation/sdks/python-machine-learning/)** - Use Apache Beam with the RunInference API to use machine learning (ML) models to do local and remote inference with batch and streaming pipelines. Follow the [RunInference API pipeline examples](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/inference) to do image classification, image segmentation, language modeling, and MNIST digit classification. See examples of [RunInference transforms](/documentation/transforms/python/elementwise/runinference/).
+*   **[Machine Learning using the RunInference API](/documentation/ml/about-ml)** - Use Apache Beam with the RunInference API to use machine learning (ML) models to do local and remote inference with batch and streaming pipelines. Follow the [RunInference API pipeline examples](https://github.com/apache/beam/tree/master/sdks/python/apache_beam/examples/inference) to do image classification, image segmentation, language modeling, and MNIST digit classification. See examples of [RunInference transforms](/documentation/transforms/python/elementwise/runinference/).
 *   **[Machine Learning Preprocessing and Prediction](https://cloud.google.com/dataflow/examples/molecules-walkthrough)** - Predict the molecular energy from data stored in the [Spatial Data File](https://en.wikipedia.org/wiki/Spatial_Data_File) (SDF) format. Train a [TensorFlow](https://www.tensorflow.org/) model with [tf.Transform](https://github.com/tensorflow/transform) for preprocessing in Python. This also shows how to create batch and streaming prediction pipelines in Apache Beam.
 *   **[Machine Learning Preprocessing](https://cloud.google.com/blog/products/ai-machine-learning/pre-processing-tensorflow-pipelines-tftransform-google-cloud)** - Find the optimal parameter settings for simulated physical machines like a bottle filler or cookie machine. The goal of each simulated machine is to have the same input/output of the actual machine, making it a "digital twin". This uses [tf.Transform](https://github.com/tensorflow/transform) for preprocessing.
 
@@ -151,11 +151,16 @@ complexity. Beam Katas are available for both Java and Python SDKs.
 
 ## Code Examples {#code-examples}
 
+### Dataflow Cookbook
+
+The [cookbook](https://github.com/GoogleCloudPlatform/dataflow-cookbook) includes examples in Java, Python, and Scala (via Scio), provides ready-to-launch and self-contained Beam pipelines.
+
 ### Java
 
 *   **[Snippets 1](https://github.com/apache/beam/tree/master/examples/java/src/main/java/org/apache/beam/examples/cookbook)** - Commonly-used data analysis patterns such as how to use [BigQuery](https://cloud.google.com/bigquery), a CombinePerKey transform, remove duplicate lines in files, filtering, joining PCollections, getting the maximum value of a PCollection, etc.
 *   **[Snippets 2](https://github.com/apache/beam/tree/master/examples/java/src/main/java/org/apache/beam/examples/common)** - Additional examples on common tasks such as configuring [BigQuery](https://cloud.google.com/bigquery), [PubSub](https://cloud.google.com/pubsub/), writing one file per window, etc.
 *   **[Complete Examples](https://github.com/apache/beam/tree/master/examples/java/src/main/java/org/apache/beam/examples/complete)** - End-to-end example pipelines such as an auto complete, a streaming word extract, calculating the Term Frequency-Inverse Document Frequency ([TF-IDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf)), getting the top Wikipedia sessions, traffic max lane flow, traffic routes, etc.
+*   **[Pub/Sub to BigQuery](https://github.com/GoogleCloudPlatform/cloud-code-samples/tree/v1/java/java-dataflow-samples/read-pubsub-write-bigquery)** - A complete example demonstrates using Apache Beam on Dataflow to convert JSON encoded Pub/Sub subscription message strings into structured data and write that data to a BigQuery table.
 
 ### Python
 
diff --git a/website/www/site/layouts/partials/section-menu/en/documentation.html b/website/www/site/layouts/partials/section-menu/en/documentation.html
index 2ab5bec69e98..dcf0b857bf97 100755
--- a/website/www/site/layouts/partials/section-menu/en/documentation.html
+++ b/website/www/site/layouts/partials/section-menu/en/documentation.html
@@ -185,6 +185,7 @@
       </ul>
     </li>
     <li><a href="/documentation/programming-guide/#batched-dofns">Batched DoFns</a></li>
+    <li><a href="/documentation/programming-guide/#transform-service">Transform service</a></li>
   </ul>
 </li>
 <li class="section-nav-item--collapsible">
diff --git a/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/adaptive-timeout-kafka.png b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/adaptive-timeout-kafka.png
new file mode 100644
index 000000000000..24626421b9f0
Binary files /dev/null and b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/adaptive-timeout-kafka.png differ
diff --git a/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/checkpoint_monitoring-history-subtasks.png b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/checkpoint_monitoring-history-subtasks.png
new file mode 100644
index 000000000000..a338db9ecdb7
Binary files /dev/null and b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/checkpoint_monitoring-history-subtasks.png differ
diff --git a/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/flink-backlog-metrics.png b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/flink-backlog-metrics.png
new file mode 100644
index 000000000000..b2dcf2bf43a4
Binary files /dev/null and b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/flink-backlog-metrics.png differ
diff --git a/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/flink-partition-assignment.png b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/flink-partition-assignment.png
new file mode 100644
index 000000000000..e607e216331b
Binary files /dev/null and b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/flink-partition-assignment.png differ
diff --git a/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/kafkaio-wait-reader.png b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/kafkaio-wait-reader.png
new file mode 100644
index 000000000000..57f49a1f0c65
Binary files /dev/null and b/website/www/site/static/images/blog/apache-beam-flink-and-kubernetes-part2/kafkaio-wait-reader.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/0-intro.png b/website/www/site/static/images/blog/scaling-streaming-workload/0-intro.png
new file mode 100644
index 000000000000..7e29be96bb68
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/0-intro.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/0-pipeline.png b/website/www/site/static/images/blog/scaling-streaming-workload/0-pipeline.png
new file mode 100644
index 000000000000..3eb0f3dfced4
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/0-pipeline.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/1-default-ps.png b/website/www/site/static/images/blog/scaling-streaming-workload/1-default-ps.png
new file mode 100644
index 000000000000..7914e7618f49
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/1-default-ps.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/1-default-throughput.png b/website/www/site/static/images/blog/scaling-streaming-workload/1-default-throughput.png
new file mode 100644
index 000000000000..3946cd9c49b6
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/1-default-throughput.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/2-skeys-ps.png b/website/www/site/static/images/blog/scaling-streaming-workload/2-skeys-ps.png
new file mode 100644
index 000000000000..61ce97e20f8a
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/2-skeys-ps.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/2-skeys-throughput.png b/website/www/site/static/images/blog/scaling-streaming-workload/2-skeys-throughput.png
new file mode 100644
index 000000000000..4f722ab79efc
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/2-skeys-throughput.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-autoscale.png b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-autoscale.png
new file mode 100644
index 000000000000..ac04649f35f9
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-autoscale.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-parallelism.png b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-parallelism.png
new file mode 100644
index 000000000000..c58afd18591c
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-parallelism.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-throughput-tr.png b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-throughput-tr.png
new file mode 100644
index 000000000000..9cb1bb36ebfb
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-throughput-tr.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-input.png b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-input.png
new file mode 100644
index 000000000000..500b9543fb16
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-input.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-output.png b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-output.png
new file mode 100644
index 000000000000..b5c70a40b9e0
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-output.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-overhead.png b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-overhead.png
new file mode 100644
index 000000000000..2d9fc977ae7d
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/3-autoshard-tr-overhead.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/4-format-transformation.png b/website/www/site/static/images/blog/scaling-streaming-workload/4-format-transformation.png
new file mode 100644
index 000000000000..751a105ef10d
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/4-format-transformation.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/4-ingestion-scale.png b/website/www/site/static/images/blog/scaling-streaming-workload/4-ingestion-scale.png
new file mode 100644
index 000000000000..dcf2bdedf8af
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/4-ingestion-scale.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/4-throughput.png b/website/www/site/static/images/blog/scaling-streaming-workload/4-throughput.png
new file mode 100644
index 000000000000..724bd9f1634b
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/4-throughput.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/5-format-trasformation.png b/website/www/site/static/images/blog/scaling-streaming-workload/5-format-trasformation.png
new file mode 100644
index 000000000000..77adacb68b3a
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/5-format-trasformation.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/5-ingestion-scale.png b/website/www/site/static/images/blog/scaling-streaming-workload/5-ingestion-scale.png
new file mode 100644
index 000000000000..04a6eaf4c3a1
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/5-ingestion-scale.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/5-input-size.png b/website/www/site/static/images/blog/scaling-streaming-workload/5-input-size.png
new file mode 100644
index 000000000000..769cec625778
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/5-input-size.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/5-output-size.png b/website/www/site/static/images/blog/scaling-streaming-workload/5-output-size.png
new file mode 100644
index 000000000000..d7b3117a4a79
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/5-output-size.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/6-format-transformation.png b/website/www/site/static/images/blog/scaling-streaming-workload/6-format-transformation.png
new file mode 100644
index 000000000000..335cc4be00ae
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/6-format-transformation.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/6-ingestion-scale.png b/website/www/site/static/images/blog/scaling-streaming-workload/6-ingestion-scale.png
new file mode 100644
index 000000000000..3a53c0db3403
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/6-ingestion-scale.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/6-latencies.png b/website/www/site/static/images/blog/scaling-streaming-workload/6-latencies.png
new file mode 100644
index 000000000000..71707af29021
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/6-latencies.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/7-current-input.png b/website/www/site/static/images/blog/scaling-streaming-workload/7-current-input.png
new file mode 100644
index 000000000000..c7db746775d4
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/7-current-input.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/7-current-shuffle-total.png b/website/www/site/static/images/blog/scaling-streaming-workload/7-current-shuffle-total.png
new file mode 100644
index 000000000000..8161f6f6ff32
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/7-current-shuffle-total.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/7-latency.png b/website/www/site/static/images/blog/scaling-streaming-workload/7-latency.png
new file mode 100644
index 000000000000..f897ec93afe5
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/7-latency.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/7-previous-data-input.png b/website/www/site/static/images/blog/scaling-streaming-workload/7-previous-data-input.png
new file mode 100644
index 000000000000..968b4f5635fb
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/7-previous-data-input.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/7-previous-shuffle-total.png b/website/www/site/static/images/blog/scaling-streaming-workload/7-previous-shuffle-total.png
new file mode 100644
index 000000000000..cfe9d61c306a
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/7-previous-shuffle-total.png differ
diff --git a/website/www/site/static/images/blog/scaling-streaming-workload/7-resources.png b/website/www/site/static/images/blog/scaling-streaming-workload/7-resources.png
new file mode 100644
index 000000000000..4950f219334f
Binary files /dev/null and b/website/www/site/static/images/blog/scaling-streaming-workload/7-resources.png differ