From 780095939d9f3a72b5a4496158ea2cb165096cae Mon Sep 17 00:00:00 2001 From: johnjcasey <95318300+johnjcasey@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:36:31 -0400 Subject: [PATCH] Feature/automate io performance tests (#28266) * Update 2.50 release notes to include new Kafka topicPattern feature * Create groovy class for io performance tests Create gradle task and github actions config for GCS using this. * delete unnecessary class * fix env call * fix call to gradle * run on hosted runner for testing * add additional checkout * add destination for triggered tests * move env variables to correct location * try uploading against separate dataset * try without a user * update branch checkout, try to view the failure log * run on failure * update to use correct BigQuery instance * convert to matrix * add result reporting * add failure clause * remove failure clause, update to run on self-hosted * address comments, clean up build * clarify branching --- ...m_PostCommit_Java_IO_Performance_Tests.yml | 114 ++++++++++++++++++ .../gradle/IoPerformanceTestUtilities.groovy | 45 +++++++ it/build.gradle | 33 +++++ it/google-cloud-platform/build.gradle | 6 +- 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml create mode 100644 buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy create mode 100644 it/build.gradle diff --git a/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml new file mode 100644 index 000000000000..ffd5751fd8b3 --- /dev/null +++ b/.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PostCommit Java IO Performance Tests + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + paths: ['it/google-cloud-platform/**','.github/workflows/beam_PostCommit_Java_IO_Performance_Tests.yml'] + issue_comment: + types: [created] + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.sender.login }}-${{ github.event.schedule }}' + cancel-in-progress: true + +env: + GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PostCommit_Java_IO_Performance_Tests: + if: | + github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || + github.event.comment.body == 'Run Java PostCommit IO Performance Tests' + runs-on: [self-hosted, ubuntu-20.04, main] + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.test_case }}) + strategy: + matrix: + job_name: ["beam_PostCommit_Java_IO_Performance_Tests"] + job_phrase: ["Run Java PostCommit IO Performance Tests"] + test_case: ["GCSPerformanceTest", "BigTablePerformanceTest"] + steps: + - uses: actions/checkout@v3 + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.test_case }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.test_case }}) + - name: Checkout release branch + if: github.event_name == 'schedule' #This has scheduled runs run against the latest release + uses: actions/checkout@v3 + with: + ref: v2.50.0 #TODO(https://github.com/apache/beam/issues/28330) automate updating this + repository: apache/beam + - name: Install Java + uses: actions/setup-java@v3.8.0 + with: + distribution: 'zulu' + java-version: '8' + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + with: + cache-read-only: false + - name: Authenticate on GCP + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_email: ${{ secrets.GCP_SA_EMAIL }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ secrets.GCP_PROJECT_ID }} + export_default_credentials: true + - name: run scheduled javaPostcommitIOPerformanceTests script + if: github.event_name == 'schedule' #This ensures only scheduled runs publish metrics publicly by changing which exportTable is configured + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :it:${{ matrix.test_case }} + env: + exportDataset: performance_tests + exportTable: io_performance_metrics + - name: run triggered javaPostcommitIOPerformanceTests script + if: github.event_name != 'schedule' + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :it:${{ matrix.test_case }} + env: + exportDataset: performance_tests + exportTable: io_performance_metrics_test \ No newline at end of file diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy new file mode 100644 index 000000000000..844afd75f008 --- /dev/null +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/IoPerformanceTestUtilities.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.gradle + +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test + +import javax.inject.Inject + +class IoPerformanceTestUtilities { + abstract static class IoPerformanceTest extends Test { + @Inject + IoPerformanceTest(Project runningProject, String module, String testClass, Map systemProperties){ + group = "Verification" + description = "Runs IO Performance Test for $testClass" + outputs.upToDateWhen { false } + testClassesDirs = runningProject.findProject(":it:${module}").sourceSets.test.output.classesDirs + classpath = runningProject.sourceSets.test.runtimeClasspath + runningProject.findProject(":it:${module}").sourceSets.test.runtimeClasspath + + include "**/${testClass}.class" + + systemProperty 'exportDataset', System.getenv('exportDataset') + systemProperty 'exportTable', System.getenv('exportTable') + + for (entry in systemProperties){ + systemProperty entry.key, entry.value + } + } + } +} diff --git a/it/build.gradle b/it/build.gradle new file mode 100644 index 000000000000..35ccbba4c360 --- /dev/null +++ b/it/build.gradle @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +plugins { id 'org.apache.beam.module' } +applyJavaNature( + automaticModuleName: 'org.apache.beam.it', +) + +description = "Apache Beam :: IT" +ext.summary = "Integration test utilities suites." + +//These registrations exist to make our matrix Github Action simple to configure +tasks.register('GCSPerformanceTest') { + dependsOn(":it:google-cloud-platform:GCSPerformanceTest") +} + +tasks.register('BigTablePerformanceTest') { + dependsOn(":it:google-cloud-platform:BigTablePerformanceTest") +} \ No newline at end of file diff --git a/it/google-cloud-platform/build.gradle b/it/google-cloud-platform/build.gradle index f43b3f25720b..0917ddd3e21a 100644 --- a/it/google-cloud-platform/build.gradle +++ b/it/google-cloud-platform/build.gradle @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import org.apache.beam.gradle.IoPerformanceTestUtilities plugins { id 'org.apache.beam.module' } applyJavaNature( @@ -74,4 +75,7 @@ dependencies { testImplementation library.java.mockito_inline testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadowTest") testRuntimeOnly library.java.slf4j_simple -} \ No newline at end of file +} + +tasks.register("GCSPerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'FileBasedIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']) +tasks.register("BigTablePerformanceTest", IoPerformanceTestUtilities.IoPerformanceTest, project, 'google-cloud-platform', 'BigTableIOLT', ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']) \ No newline at end of file