From 4da043dfb2f75b8f3803e931d451d4f799a643c7 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Wed, 3 Jul 2024 11:56:56 -0400 Subject: [PATCH] Bump to Java11 for GHA runner Bump default gradle to 8.4 on GHA runner Remove explicit ref to java8 in GHA; bump default GHA java version to 11 --- .../setup-environment-action/action.yml | 2 +- .../arc/images/Dockerfile | 16 +- .github/workflows/README.md | 18 +- ..._LoadTests_Java_GBK_Dataflow_V2_Batch.yml} | 54 ++--- ...dTests_Java_GBK_Dataflow_V2_Streaming.yml} | 52 ++--- ...PostCommit_Java_Examples_Dataflow_Java.yml | 4 +- ...tCommit_Java_Examples_Dataflow_V2_Java.yml | 4 +- ..._ValidatesRunner_Dataflow_JavaVersions.yml | 6 +- ...va_ValidatesRunner_Direct_JavaVersions.yml | 6 +- ...mmit_Java_ValidatesRunner_Flink_Java8.yml} | 21 +- ...mmit_Java_ValidatesRunner_Spark_Java8.yml} | 21 +- .github/workflows/beam_PreCommit_SQL.yml | 2 +- ...ava11.yml => beam_PreCommit_SQL_Java8.yml} | 22 +- ...Dataflow_V2_Batch_2GB_of_100B_records.txt} | 0 ...ataflow_V2_Batch_2GB_of_100kB_records.txt} | 0 ..._Dataflow_V2_Batch_2GB_of_10B_records.txt} | 0 ..._times_with_2GB_10-byte_records_total.txt} | 0 ..._times_with_2GB_10-byte_records_total.txt} | 0 ...2_Batch_reiterate_4_times_10kB_values.txt} | 0 ...V2_Batch_reiterate_4_times_2MB_values.txt} | 0 ...flow_V2_Streaming_2GB_of_100B_records.txt} | 0 ...low_V2_Streaming_2GB_of_100kB_records.txt} | 0 ...aflow_V2_Streaming_2GB_of_10B_records.txt} | 0 ..._times_with_2GB_10-byte_records_total.txt} | 0 ..._times_with_2GB_10-byte_records_total.txt} | 0 ...reaming_reiterate_4_times_10kB_values.txt} | 0 ...treaming_reiterate_4_times_2MB_values.txt} | 0 .../beam/gradle/BeamModulePlugin.groovy | 213 +++++++++--------- .../beam/runners/direct/DirectRunnerTest.java | 2 +- sdks/java/testing/test-utils/build.gradle | 2 +- .../jvmverification/JvmVerification.java | 6 + 31 files changed, 211 insertions(+), 240 deletions(-) rename .github/workflows/{beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml => beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml} (78%) rename .github/workflows/{beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml => beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml} (79%) rename .github/workflows/{beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml => beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml} (90%) rename .github/workflows/{beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml => beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml} (90%) rename .github/workflows/{beam_PreCommit_SQL_Java11.yml => beam_PreCommit_SQL_Java8.yml} (90%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt => java_GBK_Dataflow_V2_Batch_2GB_of_100B_records.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt => java_GBK_Dataflow_V2_Batch_2GB_of_100kB_records.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt => java_GBK_Dataflow_V2_Batch_2GB_of_10B_records.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt => java_GBK_Dataflow_V2_Batch_reiterate_4_times_10kB_values.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt => java_GBK_Dataflow_V2_Batch_reiterate_4_times_2MB_values.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt => java_GBK_Dataflow_V2_Streaming_2GB_of_100B_records.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt => java_GBK_Dataflow_V2_Streaming_2GB_of_100kB_records.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt => java_GBK_Dataflow_V2_Streaming_2GB_of_10B_records.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt => java_GBK_Dataflow_V2_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt => java_GBK_Dataflow_V2_Streaming_reiterate_4_times_10kB_values.txt} (100%) rename .github/workflows/load-tests-pipeline-options/{java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt => java_GBK_Dataflow_V2_Streaming_reiterate_4_times_2MB_values.txt} (100%) diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index 3a14112cf0ef..d3d26c1ad3c0 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -48,7 +48,7 @@ runs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: ${{ inputs.java-version == 'default' && '8' || inputs.java-version }} + java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }} - name: Setup Gradle uses: gradle/gradle-build-action@v2 with: diff --git a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile index 3737492f6179..c1c1955a2177 100644 --- a/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile +++ b/.github/gh-actions-self-hosted-runners/arc/images/Dockerfile @@ -37,16 +37,16 @@ RUN curl -OL https://go.dev/dl/go${go_version}.linux-amd64.tar.gz && \ rm go${go_version}.linux-amd64.tar.gz ENV PATH="${PATH}:/usr/local/go/bin" #Install Java -RUN curl -OL https://cdn.azul.com/zulu/bin/zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \ - tar -C /usr/local -xzf zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \ - rm zulu8.70.0.23-ca-jdk8.0.372-linux_x64.tar.gz && \ - mv /usr/local/zulu8.70.0.23-ca-jdk8.0.372-linux_x64 /usr/local/java +RUN curl -OL https://cdn.azul.com/zulu/bin/zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \ + tar -C /usr/local -xzf zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \ + rm zulu11.74.15-ca-jdk11.0.24-linux_x64.tar.gz && \ + mv /usr/local/zulu11.74.15-ca-jdk11.0.24-linux_x64 /usr/local/java ENV PATH="${PATH}:/usr/local/java/bin" #Install Gradle -RUN curl -OL https://services.gradle.org/distributions/gradle-7.3.3-bin.zip && \ - unzip gradle-7.3.3-bin.zip && \ - rm gradle-7.3.3-bin.zip && \ - mv gradle-7.3.3 /usr/local/gradle +RUN curl -OL https://services.gradle.org/distributions/gradle-8.4-bin.zip && \ + unzip gradle-8.4-bin.zip && \ + rm gradle-8.4-bin.zip && \ + mv gradle-8.4 /usr/local/gradle ENV PATH="${PATH}:/usr/local/gradle/bin" #Install GitHub CLI RUN curl -OL https://github.com/cli/cli/releases/download/v2.31.0/gh_2.31.0_linux_amd64.tar.gz && \ diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 93e568913f8c..0f3a42583c36 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -268,7 +268,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour | [ PreCommit RAT ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml) | N/A | `Run RAT PreCommit` | [![.github/workflows/beam_PreCommit_RAT.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_RAT.yml?query=event%3Aschedule) | | [ PreCommit Spotless ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml) | N/A | `Run Spotless PreCommit` | [![.github/workflows/beam_PreCommit_Spotless.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Spotless.yml?query=event%3Aschedule) | | [ PreCommit SQL ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml) | N/A |`Run SQL PreCommit`| [![.github/workflows/beam_PreCommit_SQL.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL.yml?query=event%3Aschedule) | -| [ PreCommit SQL Java11 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java11.yml) | N/A |`Run SQL_Java11 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java11.yml?query=event%3Aschedule) | +| [ PreCommit SQL Java8 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java8.yml) | N/A |`Run SQL_Java8 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java8.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java8.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java8.yml?query=event%3Aschedule) | | [ PreCommit SQL Java17 ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml) | N/A |`Run SQL_Java17 PreCommit`| [![.github/workflows/beam_PreCommit_SQL_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_SQL_Java17.yml?query=event%3Aschedule) | | [ PreCommit Typescript ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml) | N/A |`Run Typescript PreCommit`| [![.github/workflows/beam_PreCommit_Typescript.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Typescript.yml?query=event%3Aschedule) | | [ PreCommit Website ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml) | N/A |`Run Website PreCommit`| [![.github/workflows/beam_PreCommit_Website.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml?query=event%3Aschedule) | @@ -303,9 +303,9 @@ PostCommit Jobs run in a schedule against master branch and generally do not get | [ PostCommit Java Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml) | N/A |`beam_PostCommit_Java_DataflowV2.json`| [![.github/workflows/beam_PostCommit_Java_DataflowV2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_DataflowV2.yml?query=event%3Aschedule) | | [ PostCommit Java Examples Dataflow ARM ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml) | ['8','11','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_ARM.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml?query=event%3Aschedule) | | [ PostCommit Java Examples Dataflow](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml) | N/A |`beam_PostCommit_Java_Examples_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) | -| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['11','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml?query=event%3Aschedule) | +| [ PostCommit Java Examples Dataflow Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml) | ['8','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml?query=event%3Aschedule) | | [ PostCommit Java Examples Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml) | N/A |`beam_PostCommit_Java_Examples_Dataflow_V2.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2.yml?query=event%3Aschedule) | -| [ PostCommit Java Examples Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml) | ['11','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_V2_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml?query=event%3Aschedule) | +| [ PostCommit Java Examples Dataflow V2 Java ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml) | ['8','17','21'] |`beam_PostCommit_Java_Examples_Dataflow_V2_Java.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml?query=event%3Aschedule) | | [ PostCommit Java Examples Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml) | N/A |`beam_PostCommit_Java_Examples_Direct.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Direct.yml?query=event%3Aschedule) | | [ PostCommit Java Examples Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml) | N/A |`beam_PostCommit_Java_Examples_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Flink.yml?query=event%3Aschedule) | | [ PostCommit Java Examples Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml) | N/A |`beam_PostCommit_Java_Examples_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Examples_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Examples_Spark.yml?query=event%3Aschedule) | @@ -334,17 +334,17 @@ PostCommit Jobs run in a schedule against master branch and generally do not get | [ PostCommit Java Tpcds Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml) | N/A |`beam_PostCommit_Java_Tpcds_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Dataflow.yml?query=event%3Aschedule) | | [ PostCommit Java Tpcds Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml) | N/A |`beam_PostCommit_Java_Tpcds_Flink.json`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Flink.yml?query=event%3Aschedule) | | [ PostCommit Java Tpcds Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml) | N/A |`beam_PostCommit_Java_Tpcds_Spark.json`| [![.github/workflows/beam_PostCommit_Java_Tpcds_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_Tpcds_Spark.yml?query=event%3Aschedule) | -| [ PostCommit Java ValidatesRunner Dataflow JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml) | ['11','17'] |`beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml?query=event%3Aschedule) | +| [ PostCommit Java ValidatesRunner Dataflow JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml) | ['8','21'] |`beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Dataflow V2 Streaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2_Streaming.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Dataflow V2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_V2.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Dataflow.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow.yml?query=event%3Aschedule) | -| [ PostCommit Java ValidatesRunner Direct JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml) | ['11','17'] |`beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml?query=event%3Aschedule) | +| [ PostCommit Java ValidatesRunner Direct JavaVersions ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml) | ['8','21'] |`beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Direct ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Direct.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Direct.yml?query=event%3Aschedule) | -| [ PostCommit Java ValidatesRunner Flink Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml?query=event%3Aschedule) | +| [ PostCommit Java ValidatesRunner Flink Java8 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink_Java8.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Flink ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Flink.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Flink.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Samza ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Samza.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Samza.yml?query=event%3Aschedule) | -| [ PostCommit Java ValidatesRunner Spark Java11 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml?query=event%3Aschedule) | +| [ PostCommit Java ValidatesRunner Spark Java8 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark_Java8.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Spark ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Spark.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Spark.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner SparkStructuredStreaming ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming.yml?query=event%3Aschedule) | | [ PostCommit Java ValidatesRunner Twister2 ](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml) | N/A |`beam_PostCommit_Java_ValidatesRunner_Twister2.json`| [![.github/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PostCommit_Java_ValidatesRunner_Twister2.yml?query=event%3Aschedule) | @@ -450,9 +450,9 @@ PostCommit Jobs run in a schedule against master branch and generally do not get | [ LoadTests Java Combine SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_Combine_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule) | [ LoadTests Java GBK Dataflow Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Batch.yml?query=event%3Aschedule) | [ LoadTests Java GBK Dataflow Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_Streaming.yml?query=event%3Aschedule) -| [ LoadTests Java GBK Dataflow V2 Batch Java11 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml?query=event%3Aschedule) +| [ LoadTests Java GBK Dataflow V2 Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml?query=event%3Aschedule) | [ LoadTests Java GBK Dataflow V2 Batch Java17 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml?query=event%3Aschedule) -| [ LoadTests Java GBK Dataflow V2 Streaming Java11 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml?query=event%3Aschedule) +| [ LoadTests Java GBK Dataflow V2 Streaming ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml?query=event%3Aschedule) | [ LoadTests Java GBK Dataflow V2 Streaming Java17 ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java17.yml?query=event%3Aschedule) | [ LoadTests Java GBK Smoke ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_Smoke.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_Smoke.yml?query=event%3Aschedule) | [ LoadTests Java GBK SparkStructuredStreaming Batch ](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml) | N/A | [![.github/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_LoadTests_Java_GBK_SparkStructuredStreaming_Batch.yml?query=event%3Aschedule) diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml similarity index 78% rename from .github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml rename to .github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml index 94578af1b9a8..639381984c19 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Batch.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: LoadTests Java GBK Dataflow V2 Batch Java11 +name: LoadTests Java GBK Dataflow V2 Batch on: schedule: @@ -49,18 +49,18 @@ env: INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} jobs: - beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11: + beam_LoadTests_Java_GBK_Dataflow_V2_Batch: if: | github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || - github.event.comment.body == 'Run Load Tests Java 11 GBK Dataflow V2 Batch' + github.event.comment.body == 'Run Load Tests GBK Dataflow V2 Batch' runs-on: [self-hosted, ubuntu-20.04, main] timeout-minutes: 240 name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: matrix: - job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11"] - job_phrase: ["Run Load Tests Java 11 GBK Dataflow V2 Batch"] + job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Batch"] + job_phrase: ["Run Load Tests GBK Dataflow V2 Batch"] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -72,20 +72,20 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: 11 + java-version: default - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action @@ -95,9 +95,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_1 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -106,9 +104,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_2 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -117,9 +113,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_3 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -128,9 +122,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_4 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -139,9 +131,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_5 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -150,9 +140,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_6 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -161,6 +149,4 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_Java11_test_arguments_7 }}' \ No newline at end of file + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Batch_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml similarity index 79% rename from .github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml rename to .github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml index 8e527e4ee0d7..4a55256ab084 100644 --- a/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11.yml +++ b/.github/workflows/beam_LoadTests_Java_GBK_Dataflow_V2_Streaming.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: LoadTests Java GBK Dataflow V2 Streaming Java11 +name: LoadTests Java GBK Dataflow V2 Streaming on: schedule: @@ -49,7 +49,7 @@ env: INFLUXDB_USER_PASSWORD: ${{ secrets.INFLUXDB_USER_PASSWORD }} jobs: - beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11: + beam_LoadTests_Java_GBK_Dataflow_V2_Streaming: if: | github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || @@ -59,8 +59,8 @@ jobs: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) strategy: matrix: - job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11"] - job_phrase: ["Run Load Tests Java 11 GBK Dataflow V2 Streaming"] + job_name: ["beam_LoadTests_Java_GBK_Dataflow_V2_Streaming"] + job_phrase: ["Run Load Tests GBK Dataflow V2 Streaming"] steps: - uses: actions/checkout@v4 - name: Setup repository @@ -72,20 +72,20 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: 11 + java-version: default - name: Prepare test arguments uses: ./.github/actions/test-arguments-action with: test-type: load test-language: java argument-file-paths: | - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt - ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_10B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_100B_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_100kB_records.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_reiterate_4_times_10kB_values.txt + ${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_reiterate_4_times_2MB_values.txt # The env variables are created and populated in the test-arguments-action as "_test_arguments_" - name: run Load test 2GB of 10B records uses: ./.github/actions/gradle-command-self-hosted-action @@ -95,9 +95,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_1 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_1 }}' \ - name: run Load test 2GB of 100B records uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -106,9 +104,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_2 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_2 }}' \ - name: run Load test 2GB of 100kB records uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -117,9 +113,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_3 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_3 }}' \ - name: run Load test fanout 4 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -128,9 +122,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_4 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_4 }}' \ - name: run Load test fanout 8 times with 2GB 10-byte records total uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -139,9 +131,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_5 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_5 }}' \ - name: run Load test reiterate 4 times 10kB values uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -150,9 +140,7 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_6 }}' \ + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_6 }}' \ - name: run Load test reiterate 4 times 2MB values uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -161,6 +149,4 @@ jobs: -PloadTest.mainClass=org.apache.beam.sdk.loadtests.GroupByKeyLoadTest \ -Prunner=:runners:google-cloud-dataflow-java \ -Prunner.version=V2 \ - -PtestJavaVersion=11 \ - -Pjava11Home=$JAVA_HOME_11_X64 \ - '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_Java11_test_arguments_7 }}' \ No newline at end of file + '-PloadTest.args=${{ env.beam_LoadTests_Java_GBK_Dataflow_V2_Streaming_test_arguments_7 }}' \ No newline at end of file diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml index dee56f8e6630..13ab05f8f173 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_Java.yml @@ -60,7 +60,7 @@ jobs: matrix: job_name: [beam_PostCommit_Java_Examples_Dataflow_Java] job_phrase: [Run Java examples on Dataflow Java] - java_version: ['11','17','21'] + java_version: ['8','17','21'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || @@ -79,7 +79,7 @@ jobs: with: java-version: | ${{ matrix.java_version }} - 8 + 11 - name: run java${{ matrix.java_version }}PostCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml index 07398577ddd8..cd2486ae8e10 100644 --- a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml +++ b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_V2_Java.yml @@ -61,7 +61,7 @@ jobs: job_name: [beam_PostCommit_Java_Examples_Dataflow_V2_Java] job_phrase_1: [Run Java ] job_phrase_2: [Examples on Dataflow Runner V2] - java_version: ['11', '17', '21'] + java_version: ['8', '17', '21'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || @@ -81,7 +81,7 @@ jobs: with: java-version: | ${{ matrix.java_version }} - 8 + 11 - name: run PostCommit Java Examples Dataflow V2 Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml index e4b7643bd2d8..da2ba2f88465 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions.yml @@ -60,7 +60,7 @@ jobs: matrix: job_name: [beam_PostCommit_Java_ValidatesRunner_Dataflow_JavaVersions] job_phrase: [Run Dataflow ValidatesRunner Java] - java_version: ['11','17'] + java_version: ['8', '21'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || @@ -79,11 +79,11 @@ jobs: with: java-version: | ${{ matrix.java_version }} - 8 + 11 - name: run jar Java${{ matrix.java_version }} script run: | ./gradlew runners:google-cloud-dataflow-java:testJar :runners:google-cloud-dataflow-java:worker:shadowJar \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 + -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml index 370a0698fd34..75ebbda93f80 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions.yml @@ -60,7 +60,7 @@ jobs: matrix: job_name: [beam_PostCommit_Java_ValidatesRunner_Direct_JavaVersions] job_phrase: [Run Direct ValidatesRunner Java] - java_version: ['11','17'] + java_version: ['8', '21'] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || @@ -79,11 +79,11 @@ jobs: with: java-version: | ${{ matrix.java_version }} - 8 + 11 - name: run jar Java${{ matrix.java_version }} script run: | ./gradlew :runners:direct-java:shadowJar :runners:direct-java:shadowTestJar \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 + -Dorg.gradle.java.home=$JAVA_HOME_${{ matrix.java_version }}_X64 - name: run validatesRunner Java${{ matrix.java_version }} script uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml similarity index 90% rename from .github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml rename to .github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml index 9133b00218a3..15c99d7bfb37 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.yml @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. -name: PostCommit Java ValidatesRunner Flink Java11 +name: PostCommit Java ValidatesRunner Flink Java8 on: schedule: - cron: '45 4/6 * * *' pull_request_target: - paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java11.json'] + paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Flink_Java8.json'] workflow_dispatch: # This allows a subsequently queued workflow run to interrupt previous runs @@ -51,19 +51,19 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: - beam_PostCommit_Java_ValidatesRunner_Flink_Java11: + beam_PostCommit_Java_ValidatesRunner_Flink_Java8: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) runs-on: [self-hosted, ubuntu-20.04, main] timeout-minutes: 270 strategy: matrix: - job_name: [beam_PostCommit_Java_ValidatesRunner_Flink_Java11] - job_phrase: [Run Flink ValidatesRunner Java 11] + job_name: [beam_PostCommit_Java_ValidatesRunner_Flink_Java8] + job_phrase: [Run Flink ValidatesRunner Java 8] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || - startswith(github.event.comment.body, 'Run Flink ValidatesRunner Java 11') + startswith(github.event.comment.body, 'Run Flink ValidatesRunner Java 8') steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,13 +76,12 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: | - 11 8 + 11 - name: run jar Java8 script run: | - ./gradlew :runners:flink:1.15:jar :runners:flink:1.15:testJar \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 - - name: run validatesRunner Java11 script + ./gradlew :runners:flink:1.15:jar :runners:flink:1.15:testJar + - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:flink:1.15:validatesRunner @@ -95,7 +94,7 @@ jobs: -x testJar \ -x classes \ -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_11_X64 \ + -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml similarity index 90% rename from .github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml rename to .github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml index 40fc60fea644..c05284186617 100644 --- a/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.yml +++ b/.github/workflows/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.yml @@ -15,13 +15,13 @@ # specific language governing permissions and limitations # under the License. -name: PostCommit Java ValidatesRunner Spark Java11 +name: PostCommit Java ValidatesRunner Spark Java8 on: schedule: - cron: '45 4/6 * * *' pull_request_target: - paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java11.json'] + paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Java_ValidatesRunner_Spark_Java8.json'] workflow_dispatch: # This allows a subsequently queued workflow run to interrupt previous runs @@ -51,19 +51,19 @@ env: GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} jobs: - beam_PostCommit_Java_ValidatesRunner_Spark_Java11: + beam_PostCommit_Java_ValidatesRunner_Spark_Java8: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) runs-on: [self-hosted, ubuntu-20.04, main] timeout-minutes: 270 strategy: matrix: - job_name: [beam_PostCommit_Java_ValidatesRunner_Spark_Java11] - job_phrase: [Run Spark ValidatesRunner Java 11] + job_name: [beam_PostCommit_Java_ValidatesRunner_Spark_Java8] + job_phrase: [Run Spark ValidatesRunner Java 8] if: | github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || - startswith(github.event.comment.body, 'Run Spark ValidatesRunner Java 11') + startswith(github.event.comment.body, 'Run Spark ValidatesRunner Java 8') steps: - uses: actions/checkout@v4 - name: Setup repository @@ -76,13 +76,12 @@ jobs: uses: ./.github/actions/setup-environment-action with: java-version: | - 11 8 + 11 - name: run jar Java8 script run: | - ./gradlew :runners:spark:3:jar :runners:spark:3:testJar \ - -Dorg.gradle.java.home=$JAVA_HOME_8_X64 - - name: run validatesRunner Java11 script + ./gradlew :runners:spark:3:jar :runners:spark:3:testJar + - name: run validatesRunner Java8 script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :runners:spark:3:validatesRunner @@ -95,7 +94,7 @@ jobs: -x testJar \ -x classes \ -x testClasses \ - -Dorg.gradle.java.home=$JAVA_HOME_11_X64 \ + -Dorg.gradle.java.home=$JAVA_HOME_8_X64 \ max-workers: 12 - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/beam_PreCommit_SQL.yml b/.github/workflows/beam_PreCommit_SQL.yml index 53b21de29775..b4002fcc2a79 100644 --- a/.github/workflows/beam_PreCommit_SQL.yml +++ b/.github/workflows/beam_PreCommit_SQL.yml @@ -81,7 +81,7 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: 11 + java-version: default - name: Build and Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/beam_PreCommit_SQL_Java11.yml b/.github/workflows/beam_PreCommit_SQL_Java8.yml similarity index 90% rename from .github/workflows/beam_PreCommit_SQL_Java11.yml rename to .github/workflows/beam_PreCommit_SQL_Java8.yml index d27880ca4375..23938821b2e8 100644 --- a/.github/workflows/beam_PreCommit_SQL_Java11.yml +++ b/.github/workflows/beam_PreCommit_SQL_Java8.yml @@ -13,16 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: PreCommit SQL Java11 +name: PreCommit SQL Java8 on: push: tags: ['v*'] branches: ['master', 'release-*'] - paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL_Java11.yml'] + paths: ['sdks/java/extensions/sql/**','.github/workflows/beam_PreCommit_SQL_Java8.yml'] pull_request_target: branches: ['master', 'release-*'] - paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_SQL_Java11.json'] + paths: ['sdks/java/extensions/sql/**', 'release/trigger_all_tests.json', '.github/trigger_files/beam_PreCommit_SQL_Java8.json'] issue_comment: types: [created] schedule: @@ -56,20 +56,20 @@ permissions: statuses: read jobs: - beam_PreCommit_SQL_Java11: + beam_PreCommit_SQL_Java8: name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) runs-on: [self-hosted, ubuntu-20.04, main] strategy: matrix: - job_name: [beam_PreCommit_SQL_Java11] - job_phrase: [Run SQL_Java11 PreCommit] + job_name: [beam_PreCommit_SQL_Java8] + job_phrase: [Run SQL_Java8 PreCommit] timeout-minutes: 120 if: | github.event_name == 'push' || github.event_name == 'pull_request_target' || (github.event_name == 'schedule' && github.repository == 'apache/beam') || github.event_name == 'workflow_dispatch' || - github.event.comment.body == 'Run SQL_Java11 PreCommit' + github.event.comment.body == 'Run SQL_Java8 PreCommit' steps: - uses: actions/checkout@v4 - name: Setup repository @@ -81,7 +81,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: 11 + java-version: | + 8 + 11 python-version: default go-version: default - name: Install Flutter @@ -96,9 +98,9 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ - -PtestJavaVersion=11 \ + -PtestJavaVersion=8 \ -PskipCheckerFramework \ - -Pjava11Home=$JAVA_HOME_11_X64 \ + -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_Java11_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Batch_reiterate_4_times_2MB_values.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_100B_records.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_100B_records.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_100kB_records.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_100kB_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_100kB_records.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_10B_records.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_2GB_of_10B_records.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_2GB_of_10B_records.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_4_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_fanout_4_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_fanout_8_times_with_2GB_10-byte_records_total.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_fanout_8_times_with_2GB_10-byte_records_total.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_reiterate_4_times_10kB_values.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_10kB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_reiterate_4_times_10kB_values.txt diff --git a/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt b/.github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_reiterate_4_times_2MB_values.txt similarity index 100% rename from .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_Java11_reiterate_4_times_2MB_values.txt rename to .github/workflows/load-tests-pipeline-options/java_GBK_Dataflow_V2_Streaming_reiterate_4_times_2MB_values.txt diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index ee116423e4b0..7a4028a12a74 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -940,9 +940,27 @@ class BeamModulePlugin implements Plugin { + suffix) } + def errorProneAddModuleOpts = [ + "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" + ] + // set compiler options for java version overrides to compile with a different java version project.ext.setJavaVerOptions = { CompileOptions options, String ver -> - if (ver == '11') { + if (ver == '8') { + def java8Home = project.findProperty("java8Home") + options.fork = true + options.forkOptions.javaHome = java8Home as File + options.compilerArgs += ['-Xlint:-path'] + } else if (ver == '11') { def java11Home = project.findProperty("java11Home") options.fork = true options.forkOptions.javaHome = java11Home as File @@ -957,18 +975,7 @@ class BeamModulePlugin implements Plugin { // https://github.com/tbroyer/gradle-errorprone-plugin#jdk-16-support options.errorprone.errorproneArgs.add("-XepDisableAllChecks") // The -J prefix is needed to workaround https://github.com/gradle/gradle/issues/22747 - options.forkOptions.jvmArgs += [ - "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] + options.forkOptions.jvmArgs += errorProneAddModuleOpts.collect { '-J' + it } } else if (ver == '21') { def java21Home = project.findProperty("java21Home") options.fork = true @@ -980,18 +987,7 @@ class BeamModulePlugin implements Plugin { // Error prone requires some packages to be exported/opened for Java 17+ // Disabling checks since this property is only used for tests options.errorprone.errorproneArgs.add("-XepDisableAllChecks") - options.forkOptions.jvmArgs += [ - "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] + options.forkOptions.jvmArgs += errorProneAddModuleOpts.collect { '-J' + it } // TODO(https://github.com/apache/beam/issues/28963) // upgrade checkerFramework to enable it in Java 21 project.checkerFramework { @@ -1481,97 +1477,94 @@ class BeamModulePlugin implements Plugin { project.tasks.analyzeDependencies.enabled = false } - // Enable errorprone static analysis - project.apply plugin: 'net.ltgt.errorprone' + // errorprone requires java9+ compiler. It can be used with Java8 but then sets a java9+ errorproneJavac. + // However, the redirect ignores any task that forks and defines either a javaHome or an executable, + // see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support + // which means errorprone cannot run when gradle runs on Java11+ but serve `-testJavaVersion=8 -Pjava8Home` options + if (!(project.findProperty('testJavaVersion') == '8')) { + // Enable errorprone static analysis + project.apply plugin: 'net.ltgt.errorprone' - project.dependencies { - errorprone("com.google.errorprone:error_prone_core:$errorprone_version") - errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.2") - // At least JDk 9 compiler is required, however JDK 8 still can be used but with additional errorproneJavac - // configuration. For more details please see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support - errorproneJavac("com.google.errorprone:javac:9+181-r4173-1") - } - - project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } - - project.tasks.withType(JavaCompile) { - options.errorprone.disableWarningsInGeneratedCode = true - options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' - - // Error Prone requires some packages to be exported/opened on Java versions that support modules, - // i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403. - // The -J prefix is not needed if forkOptions.javaHome is unset, - // see http://github.com/gradle/gradle/issues/22747 - if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 - && options.forkOptions.javaHome == null) { - options.fork = true - options.forkOptions.jvmArgs += [ - "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", - "--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED" - ] + project.dependencies { + errorprone("com.google.errorprone:error_prone_core:$errorprone_version") + errorprone("jp.skypencil.errorprone.slf4j:errorprone-slf4j:0.1.2") + // At least JDk 9 compiler is required, however JDK 8 still can be used but with additional errorproneJavac + // configuration. For more details please see https://github.com/tbroyer/gradle-errorprone-plugin#jdk-8-support + if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) == 0) { + errorproneJavac("com.google.errorprone:javac:9+181-r4173-1") + } } - // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks - options.errorprone.errorproneArgs.add("-Xep:AutoValueImmutableFields:OFF") - options.errorprone.errorproneArgs.add("-Xep:AutoValueSubclassLeaked:OFF") - options.errorprone.errorproneArgs.add("-Xep:BadImport:OFF") - options.errorprone.errorproneArgs.add("-Xep:BadInstanceof:OFF") - options.errorprone.errorproneArgs.add("-Xep:BigDecimalEquals:OFF") - options.errorprone.errorproneArgs.add("-Xep:ComparableType:OFF") - options.errorprone.errorproneArgs.add("-Xep:DoNotMockAutoValue:OFF") - options.errorprone.errorproneArgs.add("-Xep:EmptyBlockTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:EmptyCatch:OFF") - options.errorprone.errorproneArgs.add("-Xep:EqualsGetClass:OFF") - options.errorprone.errorproneArgs.add("-Xep:EqualsUnsafeCast:OFF") - options.errorprone.errorproneArgs.add("-Xep:EscapedEntity:OFF") - options.errorprone.errorproneArgs.add("-Xep:ExtendsAutoValue:OFF") - options.errorprone.errorproneArgs.add("-Xep:InlineFormatString:OFF") - options.errorprone.errorproneArgs.add("-Xep:InlineMeSuggester:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidBlockTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidInlineTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidLink:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidParam:OFF") - options.errorprone.errorproneArgs.add("-Xep:InvalidThrows:OFF") - options.errorprone.errorproneArgs.add("-Xep:JavaTimeDefaultTimeZone:OFF") - options.errorprone.errorproneArgs.add("-Xep:JavaUtilDate:OFF") - options.errorprone.errorproneArgs.add("-Xep:JodaConstructors:OFF") - options.errorprone.errorproneArgs.add("-Xep:MalformedInlineTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:MissingSummary:OFF") - options.errorprone.errorproneArgs.add("-Xep:MixedMutabilityReturnType:OFF") - options.errorprone.errorproneArgs.add("-Xep:PreferJavaTimeOverload:OFF") - options.errorprone.errorproneArgs.add("-Xep:MutablePublicArray:OFF") - options.errorprone.errorproneArgs.add("-Xep:NonCanonicalType:OFF") - options.errorprone.errorproneArgs.add("-Xep:ProtectedMembersInFinalClass:OFF") - options.errorprone.errorproneArgs.add("-Xep:Slf4jFormatShouldBeConst:OFF") - options.errorprone.errorproneArgs.add("-Xep:Slf4jSignOnlyFormat:OFF") - options.errorprone.errorproneArgs.add("-Xep:StaticAssignmentInConstructor:OFF") - options.errorprone.errorproneArgs.add("-Xep:ThreadPriorityCheck:OFF") - options.errorprone.errorproneArgs.add("-Xep:TimeUnitConversionChecker:OFF") - options.errorprone.errorproneArgs.add("-Xep:UndefinedEquals:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnescapedEntity:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnnecessaryLambda:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnnecessaryMethodReference:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnnecessaryParentheses:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnrecognisedJavadocTag:OFF") - options.errorprone.errorproneArgs.add("-Xep:UnsafeReflectiveConstructionCast:OFF") - options.errorprone.errorproneArgs.add("-Xep:UseCorrectAssertInTests:OFF") - - // Sometimes a static logger is preferred, which is the convention - // currently used in beam. See docs: - // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static - options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") + project.configurations.errorprone { resolutionStrategy.force "com.google.errorprone:error_prone_core:$errorprone_version" } + + project.tasks.withType(JavaCompile) { + options.errorprone.disableWarningsInGeneratedCode = true + options.errorprone.excludedPaths = '(.*/)?(build/generated-src|build/generated.*avro-java|build/generated)/.*' + + // Error Prone requires some packages to be exported/opened on Java versions that support modules, + // i.e. Java 9 and up. The flags became mandatory in Java 17 with JEP-403. + // The -J prefix is not needed if forkOptions.javaHome is unset, + // see http://github.com/gradle/gradle/issues/22747 + if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 + && options.forkOptions.javaHome == null) { + options.fork = true + options.forkOptions.jvmArgs += errorProneAddModuleOpts + } + + // TODO(https://github.com/apache/beam/issues/20955): Enable errorprone checks + options.errorprone.errorproneArgs.add("-Xep:AutoValueImmutableFields:OFF") + options.errorprone.errorproneArgs.add("-Xep:AutoValueSubclassLeaked:OFF") + options.errorprone.errorproneArgs.add("-Xep:BadImport:OFF") + options.errorprone.errorproneArgs.add("-Xep:BadInstanceof:OFF") + options.errorprone.errorproneArgs.add("-Xep:BigDecimalEquals:OFF") + options.errorprone.errorproneArgs.add("-Xep:ComparableType:OFF") + options.errorprone.errorproneArgs.add("-Xep:DoNotMockAutoValue:OFF") + options.errorprone.errorproneArgs.add("-Xep:EmptyBlockTag:OFF") + options.errorprone.errorproneArgs.add("-Xep:EmptyCatch:OFF") + options.errorprone.errorproneArgs.add("-Xep:EqualsGetClass:OFF") + options.errorprone.errorproneArgs.add("-Xep:EqualsUnsafeCast:OFF") + options.errorprone.errorproneArgs.add("-Xep:EscapedEntity:OFF") + options.errorprone.errorproneArgs.add("-Xep:ExtendsAutoValue:OFF") + options.errorprone.errorproneArgs.add("-Xep:InlineFormatString:OFF") + options.errorprone.errorproneArgs.add("-Xep:InlineMeSuggester:OFF") + options.errorprone.errorproneArgs.add("-Xep:InvalidBlockTag:OFF") + options.errorprone.errorproneArgs.add("-Xep:InvalidInlineTag:OFF") + options.errorprone.errorproneArgs.add("-Xep:InvalidLink:OFF") + options.errorprone.errorproneArgs.add("-Xep:InvalidParam:OFF") + options.errorprone.errorproneArgs.add("-Xep:InvalidThrows:OFF") + options.errorprone.errorproneArgs.add("-Xep:JavaTimeDefaultTimeZone:OFF") + options.errorprone.errorproneArgs.add("-Xep:JavaUtilDate:OFF") + options.errorprone.errorproneArgs.add("-Xep:JodaConstructors:OFF") + options.errorprone.errorproneArgs.add("-Xep:MalformedInlineTag:OFF") + options.errorprone.errorproneArgs.add("-Xep:MissingSummary:OFF") + options.errorprone.errorproneArgs.add("-Xep:MixedMutabilityReturnType:OFF") + options.errorprone.errorproneArgs.add("-Xep:PreferJavaTimeOverload:OFF") + options.errorprone.errorproneArgs.add("-Xep:MutablePublicArray:OFF") + options.errorprone.errorproneArgs.add("-Xep:NonCanonicalType:OFF") + options.errorprone.errorproneArgs.add("-Xep:ProtectedMembersInFinalClass:OFF") + options.errorprone.errorproneArgs.add("-Xep:Slf4jFormatShouldBeConst:OFF") + options.errorprone.errorproneArgs.add("-Xep:Slf4jSignOnlyFormat:OFF") + options.errorprone.errorproneArgs.add("-Xep:StaticAssignmentInConstructor:OFF") + options.errorprone.errorproneArgs.add("-Xep:ThreadPriorityCheck:OFF") + options.errorprone.errorproneArgs.add("-Xep:TimeUnitConversionChecker:OFF") + options.errorprone.errorproneArgs.add("-Xep:UndefinedEquals:OFF") + options.errorprone.errorproneArgs.add("-Xep:UnescapedEntity:OFF") + options.errorprone.errorproneArgs.add("-Xep:UnnecessaryLambda:OFF") + options.errorprone.errorproneArgs.add("-Xep:UnnecessaryMethodReference:OFF") + options.errorprone.errorproneArgs.add("-Xep:UnnecessaryParentheses:OFF") + options.errorprone.errorproneArgs.add("-Xep:UnrecognisedJavadocTag:OFF") + options.errorprone.errorproneArgs.add("-Xep:UnsafeReflectiveConstructionCast:OFF") + options.errorprone.errorproneArgs.add("-Xep:UseCorrectAssertInTests:OFF") + + // Sometimes a static logger is preferred, which is the convention + // currently used in beam. See docs: + // https://github.com/KengoTODA/findbugs-slf4j#slf4j_logger_should_be_non_static + options.errorprone.errorproneArgs.add("-Xep:Slf4jLoggerShouldBeNonStatic:OFF") + } } // if specified test java version, modify the compile and runtime versions accordingly - if (['11', '17', '21'].contains(project.findProperty('testJavaVersion'))) { + if (['8', '11', '17', '21'].contains(project.findProperty('testJavaVersion'))) { String ver = project.getProperty('testJavaVersion') def testJavaHome = project.getProperty("java${ver}Home") diff --git a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java index 183f0d2b468e..155f5566cc83 100644 --- a/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java +++ b/runners/direct-java/src/test/java/org/apache/beam/runners/direct/DirectRunnerTest.java @@ -801,7 +801,7 @@ private static class StaticQueue implements Serializable { static class StaticQueueSource extends UnboundedSource> { - static class Checkpoint implements CheckpointMark, Serializable { + static class Checkpoint implements UnboundedSource.CheckpointMark, Serializable { final T read; diff --git a/sdks/java/testing/test-utils/build.gradle b/sdks/java/testing/test-utils/build.gradle index 6e30693d8894..334067c7d0a5 100644 --- a/sdks/java/testing/test-utils/build.gradle +++ b/sdks/java/testing/test-utils/build.gradle @@ -43,7 +43,7 @@ dependencies { testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadowTest") } -['11', '17', '21'].each { +['8', '11', '17', '21'].each { tasks.create(name: "verifyJavaVersion${it}", type: Test) { filter { includeTestsMatching "org.apache.beam.sdk.testutils.jvmverification.JvmVerification.verifyCodeIsCompiledWithJava8" diff --git a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java index a6b5d6dca6c1..9616918eca16 100644 --- a/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java +++ b/sdks/java/testing/test-utils/src/test/java/org/apache/beam/sdk/testutils/jvmverification/JvmVerification.java @@ -70,6 +70,12 @@ public void verifyTestCodeIsCompiledWithJava21() throws IOException { } // jvm + @Test + public void verifyRunningJVMVersionIs8() { + final String version = getJavaSpecification(); + assertEquals(v1_8.name, version); + } + @Test public void verifyRunningJVMVersionIs11() { final String version = getJavaSpecification();