From 9d2fea44b5982c98b626108d93275f832213b305 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Wed, 21 Feb 2024 11:10:43 -0500 Subject: [PATCH] Fix XVR test exclusions (#30354) * Fix XVR test exclusions * Add trigger files --- ..._PostCommit_Java_PVR_Spark3_Streaming.json | 0 .../beam_PostCommit_Java_PVR_Spark_Batch.json | 0 ...mmit_Java_ValidatesRunner_Dataflow_V2.json | 0 ...ValidatesRunner_Dataflow_V2_Streaming.json | 0 ...mit_Java_ValidatesRunner_Flink_Java11.json | 0 ...idatesRunner_SparkStructuredStreaming.json | 0 ...tCommit_Java_ValidatesRunner_Twister2.json | 0 ...m_PostCommit_Java_ValidatesRunner_ULR.json | 0 .../beam_PostCommit_XVR_Flink.json | 0 .../beam/gradle/BeamModulePlugin.groovy | 3 ++ runners/direct-java/build.gradle | 6 ++-- runners/flink/flink_runner.gradle | 1 + .../flink/job-server/flink_job_server.gradle | 3 +- .../google-cloud-dataflow-java/build.gradle | 4 +-- runners/jet/build.gradle | 1 + runners/portability/java/build.gradle | 1 + runners/samza/build.gradle | 1 + runners/samza/job-server/build.gradle | 3 +- .../spark/job-server/spark_job_server.gradle | 2 ++ runners/spark/spark_runner.gradle | 9 ++---- runners/twister2/build.gradle | 1 + .../beam/sdk/testing/UsesExternalService.java | 28 +++++++++++++++++++ .../sdk/testing/UsesJavaExpansionService.java | 2 +- .../testing/UsesPythonExpansionService.java | 2 +- 24 files changed, 49 insertions(+), 18 deletions(-) create mode 100644 .github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json create mode 100644 .github/trigger_files/beam_PostCommit_XVR_Flink.json create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark3_Streaming.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json b/.github/trigger_files/beam_PostCommit_Java_PVR_Spark_Batch.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Twister2.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json b/.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_ULR.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/trigger_files/beam_PostCommit_XVR_Flink.json b/.github/trigger_files/beam_PostCommit_XVR_Flink.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 6166085fc8b7..5b73940b99dd 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -2680,6 +2680,7 @@ class BeamModulePlugin implements Plugin { project.evaluationDependsOn(":sdks:python") project.evaluationDependsOn(":sdks:java:testing:expansion-service") + project.evaluationDependsOn(":sdks:java:core") project.evaluationDependsOn(":sdks:java:extensions:python") project.evaluationDependsOn(":sdks:go:test") @@ -2744,9 +2745,11 @@ class BeamModulePlugin implements Plugin { systemProperty "expansionPort", port systemProperty "semiPersistDir", config.semiPersistDir classpath = config.classpath + project.files( + project.project(":sdks:java:core").sourceSets.test.runtimeClasspath, project.project(":sdks:java:extensions:python").sourceSets.test.runtimeClasspath ) testClassesDirs = project.files( + project.project(":sdks:java:core").sourceSets.test.output.classesDirs, project.project(":sdks:java:extensions:python").sourceSets.test.output.classesDirs ) maxParallelForks config.numParallelTests diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle index b7f74dc3e538..c357b8a04328 100644 --- a/runners/direct-java/build.gradle +++ b/runners/direct-java/build.gradle @@ -142,8 +142,7 @@ task needsRunnerTests(type: Test) { excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" // MetricsPusher isn't implemented in direct runner excludeCategories "org.apache.beam.sdk.testing.UsesMetricsPusher" - excludeCategories "org.apache.beam.sdk.testing.UsesJavaExpansionService" - excludeCategories "org.apache.beam.sdk.testing.UsesPythonExpansionService" + excludeCategories "org.apache.beam.sdk.testing.UsesExternalService" excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer' } testLogging { @@ -173,8 +172,7 @@ task validatesRunner(type: Test) { excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" excludeCategories 'org.apache.beam.sdk.testing.UsesMetricsPusher' - excludeCategories "org.apache.beam.sdk.testing.UsesJavaExpansionService" - excludeCategories "org.apache.beam.sdk.testing.UsesPythonExpansionService" + excludeCategories "org.apache.beam.sdk.testing.UsesExternalService" // https://github.com/apache/beam/issues/18499 excludeCategories 'org.apache.beam.sdk.testing.UsesLoopingTimer' } diff --git a/runners/flink/flink_runner.gradle b/runners/flink/flink_runner.gradle index d8bfcb3f533a..1d91284a3d16 100644 --- a/runners/flink/flink_runner.gradle +++ b/runners/flink/flink_runner.gradle @@ -279,6 +279,7 @@ def createValidatesRunnerTask(Map m) { excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream' } else { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' // Should be run only in a properly configured SDK harness environment excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer' diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle index ab3f726e6ad9..e88b3c7d9e81 100644 --- a/runners/flink/job-server/flink_job_server.gradle +++ b/runners/flink/job-server/flink_job_server.gradle @@ -167,8 +167,7 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean checkpoi // Larger keys are possible, but they require more memory. excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above10MB' excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics' - excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService' - excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging' excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage' excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics' diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle index 27b1ca83a9b9..4c4b58d90323 100644 --- a/runners/google-cloud-dataflow-java/build.gradle +++ b/runners/google-cloud-dataflow-java/build.gradle @@ -180,8 +180,7 @@ def commonLegacyExcludeCategories = [ 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment', 'org.apache.beam.sdk.testing.LargeKeys$Above10MB', 'org.apache.beam.sdk.testing.UsesAttemptedMetrics', - 'org.apache.beam.sdk.testing.UsesJavaExpansionService', - 'org.apache.beam.sdk.testing.UsesPythonExpansionService', + 'org.apache.beam.sdk.testing.UsesExternalService', 'org.apache.beam.sdk.testing.UsesDistributionMetrics', 'org.apache.beam.sdk.testing.UsesGaugeMetrics', 'org.apache.beam.sdk.testing.UsesMultimapState', @@ -192,6 +191,7 @@ def commonLegacyExcludeCategories = [ ] def commonRunnerV2ExcludeCategories = [ + 'org.apache.beam.sdk.testing.UsesExternalService', 'org.apache.beam.sdk.testing.UsesGaugeMetrics', 'org.apache.beam.sdk.testing.UsesSetState', 'org.apache.beam.sdk.testing.UsesMapState', diff --git a/runners/jet/build.gradle b/runners/jet/build.gradle index 2b6bf2bcdff4..56a001a2bceb 100644 --- a/runners/jet/build.gradle +++ b/runners/jet/build.gradle @@ -71,6 +71,7 @@ task validatesRunnerBatch(type: Test) { useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' // Should be run only in a properly configured SDK harness environment + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories "org.apache.beam.sdk.testing.LargeKeys\$Above100MB" excludeCategories 'org.apache.beam.sdk.testing.UsesTimerMap' diff --git a/runners/portability/java/build.gradle b/runners/portability/java/build.gradle index 45709a0dbd1e..9830b48c83ad 100644 --- a/runners/portability/java/build.gradle +++ b/runners/portability/java/build.gradle @@ -149,6 +149,7 @@ def createUlrValidatesRunnerTask = { name, environmentType, dockerImageTask = "" useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' // Should be run only in a properly configured SDK harness environment + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics' excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration' diff --git a/runners/samza/build.gradle b/runners/samza/build.gradle index 8e2cc2a5eb7d..a50e0d62e59a 100644 --- a/runners/samza/build.gradle +++ b/runners/samza/build.gradle @@ -124,6 +124,7 @@ tasks.register("validatesRunner", Test) { useJUnit { includeCategories 'org.apache.beam.sdk.testing.NeedsRunner' includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' // Should be run only in a properly configured SDK harness environment excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo' diff --git a/runners/samza/job-server/build.gradle b/runners/samza/job-server/build.gradle index c9401a8aff17..7bb1b84dbaae 100644 --- a/runners/samza/job-server/build.gradle +++ b/runners/samza/job-server/build.gradle @@ -86,8 +86,7 @@ def portableValidatesRunnerTask(String name, boolean docker) { // Larger keys are possible, but they require more memory. excludeCategories 'org.apache.beam.sdk.testing.LargeKeys$Above10MB' excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics' - excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService' - excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging' excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage' excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics' diff --git a/runners/spark/job-server/spark_job_server.gradle b/runners/spark/job-server/spark_job_server.gradle index 8945ad3498c7..6c884e8e2233 100644 --- a/runners/spark/job-server/spark_job_server.gradle +++ b/runners/spark/job-server/spark_job_server.gradle @@ -108,6 +108,7 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean docker, testCategories = { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' // Should be run only in a properly configured SDK harness environment excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' @@ -167,6 +168,7 @@ def portableValidatesRunnerTask(String name, boolean streaming, boolean docker, // Batch testCategories = { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' // Should be run only in a properly configured SDK harness environment excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle index d775cfd4d6e9..5d0a7f02d17f 100644 --- a/runners/spark/spark_runner.gradle +++ b/runners/spark/spark_runner.gradle @@ -274,8 +274,7 @@ def applyBatchValidatesRunnerSetup = { Test it -> // SDF excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo' // Portability - excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService' - excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer' // Ordering excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderedDelivery' @@ -349,8 +348,7 @@ def validatesRunnerStreaming = tasks.register("validatesRunnerStreaming", Test) excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo' excludeCategories 'org.apache.beam.sdk.testing.UsesBoundedSplittableParDo' // Portability - excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService' - excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer' // Ordering excludeCategories 'org.apache.beam.sdk.testing.UsesPerKeyOrderedDelivery' @@ -405,8 +403,7 @@ tasks.register("validatesStructuredStreamingRunnerBatch", Test) { // SDF excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo' // Portability - excludeCategories 'org.apache.beam.sdk.testing.UsesJavaExpansionService' - excludeCategories 'org.apache.beam.sdk.testing.UsesPythonExpansionService' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer' excludeCategories 'org.apache.beam.sdk.testing.UsesTriggeredSideInputs' } diff --git a/runners/twister2/build.gradle b/runners/twister2/build.gradle index 36a044737d82..744c4e171fd1 100644 --- a/runners/twister2/build.gradle +++ b/runners/twister2/build.gradle @@ -80,6 +80,7 @@ def validatesRunnerBatch = tasks.register("validatesRunnerBatch", Test) { forkEvery 1 useJUnit { includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner' + excludeCategories 'org.apache.beam.sdk.testing.UsesExternalService' // Should be run only in a properly configured SDK harness environment excludeCategories 'org.apache.beam.sdk.testing.UsesSdkHarnessEnvironment' excludeCategories 'org.apache.beam.sdk.testing.FlattenWithHeterogeneousCoders' diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java new file mode 100644 index 000000000000..a9e0b9d2236c --- /dev/null +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesExternalService.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.testing; + +import org.apache.beam.sdk.annotations.Internal; + +/** + * Category tag for tests which relies on a pre-defined port, such as expansion service or transform + * service. Tests tagged with {@link UsesExternalService} should initialize such port before the + * test execution. + */ +@Internal +public interface UsesExternalService {} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java index ad919ae4b88f..766854c8cafd 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesJavaExpansionService.java @@ -24,4 +24,4 @@ * UsesJavaExpansionService} should be run for runners which support cross-language transforms. */ @Internal -public interface UsesJavaExpansionService {} +public interface UsesJavaExpansionService extends UsesExternalService {} diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java index b92742e5db8b..0fbab223934d 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/UsesPythonExpansionService.java @@ -24,4 +24,4 @@ * UsesPythonExpansionService} should be run for runners which support cross-language transforms. */ @Internal -public interface UsesPythonExpansionService {} +public interface UsesPythonExpansionService extends UsesExternalService {}