Merge branch 'master' into beam_PostCommit_Python_MongoDBIO_IT

apache · Sep 15, 2023 · d3b3f3e · d3b3f3e
2 parents d2444cc + 67a9cf3
commit d3b3f3e
Show file tree

Hide file tree

Showing 40 changed files with 2,120 additions and 117 deletions.
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml b/.github/workflows/beam_PostCommit_Java_Examples_Dataflow_ARM.yml
@@ -84,7 +84,7 @@ jobs:
         with:
           comment_phrase: ${{ matrix.job_phrase }} ${{matrix.java_version}}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) ${{matrix.java_version}}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{matrix.java_version}})
       - name: Setup self-hosted
         uses: ./.github/actions/setup-self-hosted-action
         with:

diff --git a/...tCommit_Java_Examples_Dataflow_Java11.yml → ...ostCommit_Java_Examples_Dataflow_Java.yml b/...tCommit_Java_Examples_Dataflow_Java11.yml → ...ostCommit_Java_Examples_Dataflow_Java.yml
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: PostCommit Java Examples Dataflow Java11
+name: PostCommit Java Examples Dataflow Java
 
 on:
   issue_comment:
@@ -51,40 +51,39 @@ env:
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
-  beam_PostCommit_Java_Examples_Dataflow_Java11:
-    name: ${{matrix.job_name}} (${{matrix.job_phrase}})
+  beam_PostCommit_Java_Examples_Dataflow_Java:
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }})
     runs-on: [self-hosted, ubuntu-20.04, main]
     timeout-minutes: 180
     strategy:
       fail-fast: false
       matrix:
-        job_name: [beam_PostCommit_Java_Examples_Dataflow_Java11]
-        job_phrase: [Run Java examples on Dataflow Java 11]
+        job_name: [beam_PostCommit_Java_Examples_Dataflow_Java]
+        job_phrase: [Run Java examples on Dataflow Java]
+        java_version: ['11','17']
     if: |
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'schedule' ||
-      github.event.comment.body == 'Run Java examples on Dataflow Java 11'
+      startswith(github.event.comment.body, 'Run Java examples on Dataflow Java')
     steps:
       - uses: actions/checkout@v3
       - name: Setup repository
         uses: ./.github/actions/setup-action
         with:
-          comment_phrase: ${{ matrix.job_phrase }}
+          comment_phrase: ${{ matrix.job_phrase }} ${{matrix.java_version}}
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
-      # The test requires Java 11 and Java 8 versions.
-      # Java 8 is installed second because JAVA_HOME needs to point to Java 8.
-      - name: Set up Java
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.java_version }})
+      - name: Set up Java${{ matrix.java_version }}
         uses: actions/[email protected]
         with:
           distribution: 'temurin'
           java-version: |
-            11
+            ${{ matrix.java_version }}
             8
-      - name: run PostCommit Java Examples Dataflow Java11 script
+      - name: run java${{ matrix.java_version }}PostCommit script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :runners:google-cloud-dataflow-java:examples:java11PostCommit
+          gradle-command: :runners:google-cloud-dataflow-java:examples:java${{ matrix.java_version }}PostCommit
           max-workers: 12
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v3

diff --git a/...tCommit_Java_Examples_Dataflow_Java17.yml → .../beam_PostCommit_Java_Examples_Direct.yml b/...tCommit_Java_Examples_Dataflow_Java17.yml → .../beam_PostCommit_Java_Examples_Direct.yml
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: PostCommit Java Examples Dataflow Java17
+name: PostCommit Java Examples Direct
 
 on:
   issue_comment:
@@ -51,18 +51,18 @@ env:
   GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
 
 jobs:
-  beam_PostCommit_Java_Examples_Dataflow_Java17:
+  beam_PostCommit_Java_Examples_Direct:
     name: ${{matrix.job_name}} (${{matrix.job_phrase}})
     runs-on: [self-hosted, ubuntu-20.04, main]
-    timeout-minutes: 180
+    timeout-minutes: 120
     strategy:
       matrix:
-        job_name: [beam_PostCommit_Java_Examples_Dataflow_Java17]
-        job_phrase: [Run Java examples on Dataflow Java 17]
+        job_name: [beam_PostCommit_Java_Examples_Direct]
+        job_phrase: [Run Java Examples_Direct]
     if: |
       github.event_name == 'workflow_dispatch' ||
       github.event_name == 'schedule' ||
-      github.event.comment.body == 'Run Java examples on Dataflow Java 17'
+      github.event.comment.body == 'Run Java Examples_Direct'
     steps:
       - uses: actions/checkout@v3
       - name: Setup repository
@@ -71,18 +71,15 @@ jobs:
           comment_phrase: ${{ matrix.job_phrase }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
-      - name: Set up Java
+      - name: Install Java
         uses: actions/[email protected]
         with:
-          distribution: 'temurin'
-          java-version: |
-            17
-            8
-      - name: run PostCommit Java  Examples Dataflow Java17 script
+          distribution: 'zulu'
+          java-version: '8'
+      - name: run examplesIntegrationTest script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :runners:google-cloud-dataflow-java:examples:java17PostCommit
-          max-workers: 12
+          gradle-command: :runners:direct:examplesIntegrationTest
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v3
         if: failure()

diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml b/.github/workflows/beam_PostCommit_Java_Examples_Flink.yml
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PostCommit Java Examples Flink
+
+on:
+  issue_comment:
+    types: [created]
+  schedule:
+    - cron: '0 */6 * * *'
+  workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login}}'
+  cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: write
+  checks: write
+  contents: read
+  deployments: read
+  id-token: none
+  issues: write
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+env:
+  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PostCommit_Java_Examples_Flink:
+    name: ${{matrix.job_name}} (${{matrix.job_phrase}})
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    timeout-minutes: 120
+    strategy:
+      matrix:
+        job_name: [beam_PostCommit_Java_Examples_Flink]
+        job_phrase: [Run Java Examples_Flink]
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'schedule' ||
+      github.event.comment.body == 'Run Java Examples_Flink'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Install Java
+        uses: actions/[email protected]
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+        with:
+          cache-read-only: false
+      - name: run examplesIntegrationTest script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :runners:flink:1.15:examplesIntegrationTest
+      - name: Archive JUnit Test Results
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: JUnit Test Results
+          path: "**/build/reports/tests/"
+      - name: Publish JUnit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: '**/build/test-results/**/*.xml'
diff --git a/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml b/.github/workflows/beam_PostCommit_Java_Examples_Spark.yml
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: PostCommit Java Examples Spark
+
+on:
+  issue_comment:
+    types: [created]
+  schedule:
+    - cron: '0 */6 * * *'
+  workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.body || github.event.sender.login}}'
+  cancel-in-progress: true
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: write
+  checks: write
+  contents: read
+  deployments: read
+  id-token: none
+  issues: write
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+env:
+  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PostCommit_Java_Examples_Spark:
+    name: ${{matrix.job_name}} (${{matrix.job_phrase}})
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    timeout-minutes: 120
+    strategy:
+      matrix:
+        job_name: [beam_PostCommit_Java_Examples_Spark]
+        job_phrase: [Run Java Examples_Spark]
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'schedule' ||
+      github.event.comment.body == 'Run Java Examples_Spark'
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Install Java
+        uses: actions/[email protected]
+        with:
+          distribution: 'zulu'
+          java-version: '8'
+      - name: run examplesIntegrationTest script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :runners:spark:3:examplesIntegrationTest
+      - name: Archive JUnit Test Results
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: JUnit Test Results
+          path: "**/build/reports/tests/"
+      - name: Publish JUnit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: '**/build/test-results/**/*.xml'