From ce93e1982f6e00c5367950d9748cb44116b3e607 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 14 May 2024 08:47:00 +0300 Subject: [PATCH] [improve][ci][branch-3.0] Upgrade actions in pulsar-ci and pulsar-ci-flaky, port owasp cache change (cherry picked from commit db40c8f31a8eeb8d8f5e7c4daca6234cfea116ac) --- .github/actions/upload-coverage/action.yml | 8 +- .github/changes-filter.yaml | 4 +- .github/workflows/ci-go-functions.yaml | 6 +- .github/workflows/pulsar-ci-flaky.yaml | 14 +-- .github/workflows/pulsar-ci.yaml | 126 ++++++++++++--------- 5 files changed, 90 insertions(+), 68 deletions(-) diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml index a9706e77333cb..0ba73e94a8389 100644 --- a/.github/actions/upload-coverage/action.yml +++ b/.github/actions/upload-coverage/action.yml @@ -51,7 +51,7 @@ runs: - name: "Upload to Codecov (attempt #1)" id: codecov-upload-1 if: steps.repo-check.outputs.passed == 'true' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 continue-on-error: true with: flags: ${{ inputs.flags }} @@ -64,7 +64,7 @@ runs: - name: "Upload to Codecov (attempt #2)" id: codecov-upload-2 if: steps.codecov-upload-1.outcome == 'failure' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 continue-on-error: true with: flags: ${{ inputs.flags }} @@ -77,7 +77,7 @@ runs: - name: "Upload to Codecov (attempt #3)" id: codecov-upload-3 if: steps.codecov-upload-2.outcome == 'failure' - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 # fail on last attempt continue-on-error: false with: @@ -97,4 +97,4 @@ runs: [Code coverage report](https://app.codecov.io/github/$GITHUB_REPOSITORY/commit/${head_sha}/tree) - EOF \ No newline at end of file + EOF diff --git a/.github/changes-filter.yaml b/.github/changes-filter.yaml index 250ebf692f6e6..66e5db32d4c47 100644 --- a/.github/changes-filter.yaml +++ b/.github/changes-filter.yaml @@ -12,10 +12,12 @@ docs: - 'deployment/**' - 'wiki/**' - 'pip/**' +java_non_tests: + - '**/src/main/java/**/*.java' tests: - added|modified: '**/src/test/java/**/*.java' need_owasp: - 'pom.xml' - '**/pom.xml' - 'src/owasp-dependency-check-false-positives.xml' - - 'src/owasp-dependency-check-suppressions.xml' \ No newline at end of file + - 'src/owasp-dependency-check-suppressions.xml' diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 08c287b04b557..d7daf012f686f 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -43,7 +43,7 @@ jobs: docs_only: ${{ steps.check_changes.outputs.docs_only }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Detect changed files id: changes @@ -80,13 +80,13 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} id: go diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index 86fc41f0d8d10..27d2e9895ef7e 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -54,7 +54,7 @@ jobs: collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Detect changed files id: changes @@ -101,7 +101,7 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -115,7 +115,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -126,7 +126,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -156,7 +156,7 @@ jobs: - name: Upload Jacoco report files to build artifacts if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-flaky path: target/jacoco_test_coverage_report_flaky.zip @@ -176,7 +176,7 @@ jobs: annotate_only: 'true' - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: Unit-BROKER_FLAKY-surefire-reports @@ -184,7 +184,7 @@ jobs: retention-days: 7 - name: Upload possible heap dump - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Unit-BROKER_FLAKY-heapdump diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 5bba36c1d9caa..2ba58ba102bf9 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -56,7 +56,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Detect changed files id: changes @@ -102,7 +102,7 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -116,7 +116,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -127,7 +127,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -210,7 +210,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -224,7 +224,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -235,7 +235,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.jdk || '17' }} @@ -277,7 +277,7 @@ jobs: annotate_only: 'true' - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: Unit-${{ matrix.group }}-surefire-reports @@ -285,7 +285,7 @@ jobs: retention-days: 7 - name: Upload possible heap dump, core dump or crash files - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Unit-${{ matrix.group }}-dumps @@ -314,7 +314,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -328,7 +328,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -339,7 +339,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.jdk || '17' }} @@ -361,7 +361,7 @@ jobs: zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-unittests path: target/jacoco_test_coverage_report_unittests.zip @@ -394,7 +394,7 @@ jobs: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -408,7 +408,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -419,7 +419,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -509,7 +509,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -523,7 +523,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -534,7 +534,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -558,7 +558,7 @@ jobs: ${{ matrix.setup }} - name: Set up runtime JDK ${{ matrix.runtime_jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 if: ${{ matrix.runtime_jdk }} with: distribution: 'temurin' @@ -591,7 +591,7 @@ jobs: annotate_only: 'true' - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: Integration-${{ matrix.group }}-surefire-reports @@ -599,7 +599,7 @@ jobs: retention-days: 7 - name: Upload container logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} continue-on-error: true with: @@ -625,7 +625,7 @@ jobs: PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -639,7 +639,7 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -650,7 +650,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -677,7 +677,7 @@ jobs: zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-inttests path: target/jacoco_test_coverage_report_inttests.zip @@ -712,7 +712,7 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -734,7 +734,7 @@ jobs: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -753,7 +753,7 @@ jobs: mode: full - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -765,7 +765,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -873,7 +873,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -891,7 +891,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -903,7 +903,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -953,7 +953,7 @@ jobs: annotate_only: 'true' - name: Upload container logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} continue-on-error: true with: @@ -962,7 +962,7 @@ jobs: retention-days: 7 - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: System-${{ matrix.name }}-surefire-reports @@ -988,7 +988,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -1002,7 +1002,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -1014,7 +1014,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -1040,7 +1040,7 @@ jobs: zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true - name: Upload Jacoco report files to build artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Jacoco-coverage-report-systests path: target/jacoco_test_coverage_report_systests.zip @@ -1086,7 +1086,7 @@ jobs: steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -1104,7 +1104,7 @@ jobs: limit-access-to-actor: true - name: Cache local Maven repository - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -1116,7 +1116,7 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -1158,7 +1158,7 @@ jobs: annotate_only: 'true' - name: Upload container logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} continue-on-error: true with: @@ -1167,7 +1167,7 @@ jobs: retention-days: 7 - name: Upload Surefire reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ !success() }} with: name: System-${{ matrix.name }}-surefire-reports @@ -1195,7 +1195,7 @@ jobs: if: ${{ needs.preconditions.outputs.docs_only != 'true' }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -1217,13 +1217,13 @@ jobs: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | @@ -1234,7 +1234,7 @@ jobs: ${{ runner.os }}-m2-dependencies-all- - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: 17 @@ -1252,7 +1252,7 @@ jobs: GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM uses: ./.github/actions/tune-runner-vm @@ -1266,17 +1266,19 @@ jobs: limit-access-to-actor: true - name: Cache Maven dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 timeout-minutes: 5 with: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar + !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - name: Set up JDK ${{ matrix.jdk || '17' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ matrix.jdk || '17' }} @@ -1291,6 +1293,24 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v4 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. - name: trigger dependency check run: | @@ -1298,7 +1318,7 @@ jobs: -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' - name: Upload report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ cancelled() || failure() }} continue-on-error: true with: @@ -1349,7 +1369,7 @@ jobs: - name: checkout if: ${{ needs.preconditions.outputs.docs_only != 'true' }} - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Tune Runner VM if: ${{ needs.preconditions.outputs.docs_only != 'true' }}