Merge pull request apache#31156: Reapply "Add Redistribute transform …

…to Java SDK"

.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Batch.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Docker.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Flink_Streaming.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Samza.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Batch.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Direct.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Samza.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
+}

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json

@@ -1,3 +1,4 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "https://github.com/apache/beam/pull/31156": "noting that PR #31156 should run this test"
 }

runners/flink/flink_runner.gradle

@@ -332,6 +332,9 @@ def createValidatesRunnerTask(Map m) {
           // FlinkBatchExecutionInternalTimeService does not support timer registration on timer firing.
           excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoTest$TimestampTests.testOnTimerTimestampSkew'
         }
+        // Extremely flaky: https://github.com/apache/beam/issues/19814
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful'
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful'
       }
     }
   }

runners/flink/job-server/flink_job_server.gradle

@@ -197,6 +197,8 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean checkpoi
     testFilter: {
       // Flink reshuffle override does not preserve all metadata
       excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
+      // TODO(https://github.com/apache/beam/issues/31231)
+      excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata'
       // TODO(https://github.com/apache/beam/issues/20269)
       excludeTestsMatching 'org.apache.beam.sdk.transforms.FlattenTest.testFlattenWithDifferentInputAndOutputCoders2'
       // TODO(https://github.com/apache/beam/issues/20843)

runners/portability/java/build.gradle

@@ -209,6 +209,8 @@ def createUlrValidatesRunnerTask = { name, environmentType, dockerImageTask = ""
 
       // TODO(https://github.com/apache/beam/issues/29973)
       excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
+      // TODO(https://github.com/apache/beam/issues/31231)
+      excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata'
 
       for (String test : sickbayTests) {
         excludeTestsMatching test

runners/samza/job-server/build.gradle

@@ -183,6 +183,8 @@ def portableValidatesRunnerTask(String name, boolean docker) {
             excludeTestsMatching 'org.apache.beam.sdk.transforms.GroupByKeyTest$BasicTests.testAfterProcessingTimeContinuationTriggerUsingState'
             // TODO(https://github.com/apache/beam/issues/29973)
             excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
+            // TODO(https://github.com/apache/beam/issues/31231)
+            excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata'
         }
   )
 }

runners/spark/job-server/spark_job_server.gradle

@@ -162,6 +162,14 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean docker,
         excludeTestsMatching 'CombineTest$BasicTests.testHotKeyCombining'
         // TODO(https://github.com/apache/beam/issues/29973)
         excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
+        // TODO(https://github.com/apache/beam/issues/31231)
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata'
+        // TODO(https://github.com/apache/beam/issues/31234) same reason as GroupByKeyTest and ReshuffleTest above
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributeAfterFixedWindows'
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributeAfterSlidingWindows'
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributeAfterFixedWindowsAndGroupByKey'
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributeAfterSessionsAndGroupByKey'
+        excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributeAfterSlidingWindowsAndGroupByKey'
       }
     }
     else {

runners/spark/spark_runner.gradle

@@ -286,6 +286,11 @@ def applyBatchValidatesRunnerSetup = { Test it ->
 def validatesRunnerBatch = tasks.register("validatesRunnerBatch", Test) {
   applyBatchValidatesRunnerSetup(it)
   systemProperties sparkTestProperties(["--enableSparkMetricSinks":"false"])
+
+  // TODO(https://github.com/apache/beam/issues/31231
+  it.filter {
+    excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata'
+  }
 }
 
 def validatesRunnerBatchWithBoundedSDFExperiment = tasks.register("validatesRunnerBatchWithBoundedSDFExperiment", Test) {
@@ -322,6 +327,8 @@ def validatesRunnerStreaming = tasks.register("validatesRunnerStreaming", Test)
       excludeTestsMatching 'org.apache.beam.sdk.transforms.ViewTest.testWindowedSideInputNotPresent'
       // TODO(https://github.com/apache/beam/issues/29973)
       excludeTestsMatching 'org.apache.beam.sdk.transforms.ReshuffleTest.testReshufflePreservesMetadata'
+      // TODO(https://github.com/apache/beam/issues/31231
+      excludeTestsMatching 'org.apache.beam.sdk.transforms.RedistributeTest.testRedistributePreservesMetadata'
     }
 
     // TestStream using processing time is not supported in Spark