Components Conformance Tests #515
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ------------------------------------------------------------ | |
| # Copyright 2021 The Dapr Authors | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # ------------------------------------------------------------ | |
| name: Components Conformance Tests | |
| on: | |
| repository_dispatch: | |
| types: [conformance-test] | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 */8 * * *' | |
| push: | |
| branches: | |
| - 'release-*' | |
| - 'gh-readonly-queue/master/*' | |
| pull_request: | |
| branches: | |
| - 'master' | |
| - 'release-*' | |
| - 'gh-readonly-queue/master/*' | |
| merge_group: | |
| jobs: | |
| # Based on whether this is a PR or a scheduled run, we will run a different | |
| # subset of the conformance tests. This allows all the tests not requiring | |
| # secrets to be executed on pull requests. | |
| generate-matrix: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Parse repository_dispatch payload | |
| if: github.event_name == 'repository_dispatch' | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
| echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
| echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| ref: ${{ env.CHECKOUT_REF }} | |
| - name: Generate test matrix | |
| id: generate-matrix | |
| env: | |
| VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }} | |
| run: | | |
| if [ -z "$VAULT_NAME" ]; then | |
| # Do not include cloud tests when credentials are not available | |
| node .github/scripts/test-info.mjs conformance false | |
| else | |
| # Include cloud tests | |
| node .github/scripts/test-info.mjs conformance true | |
| fi | |
| - name: Create PR comment | |
| if: env.PR_NUMBER != '' | |
| uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba | |
| with: | |
| header: ${{ github.run_id }} | |
| number: ${{ env.PR_NUMBER }} | |
| GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }} | |
| message: | | |
| # Components conformance test | |
| 🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})** | |
| Commit ref: ${{ env.CHECKOUT_REF }} | |
| outputs: | |
| test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }} | |
| conformance: | |
| name: ${{ matrix.component }} conformance | |
| # Add "id-token" with the intended permissions. | |
| # Needed by the 'Authenticate to Google Cloud' step. | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-22.04 | |
| env: | |
| UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}} | |
| GOCOV_VER: "v1.1.0" | |
| GOTESTSUM_VER: "v1.9.0" | |
| defaults: | |
| run: | |
| shell: bash | |
| needs: | |
| - generate-matrix | |
| strategy: | |
| fail-fast: false # Keep running even if one component fails | |
| matrix: | |
| include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }} | |
| steps: | |
| - name: Set default payload repo and ref | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV | |
| - name: Parse repository_dispatch payload | |
| if: github.event_name == 'repository_dispatch' | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
| echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV | |
| echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
| echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Check out code | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ env.CHECKOUT_REPO }} | |
| ref: ${{ env.CHECKOUT_REF }} | |
| - name: Setup test environment | |
| run: | | |
| # Output file | |
| echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV | |
| # Current time (used by Terraform) | |
| echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV} | |
| - name: Configure conformance test and source path | |
| run: | | |
| TEST_COMPONENT=$(echo ${{ matrix.component }} | sed -E 's/\./\//g') | |
| export SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}" | |
| echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV | |
| # converts slashes to dots in this string, so that it doesn't consider them sub-folders | |
| export SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" |sed 's#/#\.#g') | |
| echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV | |
| - uses: Azure/login@v1 | |
| if: matrix.required-secrets != '' | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| # Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal: | |
| # az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID | |
| # Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated | |
| - name: Setup secrets | |
| if: matrix.required-secrets != '' | |
| env: | |
| VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }} | |
| run: | | |
| secrets="${{ matrix.required-secrets }}" | |
| for secretName in $(echo -n $secrets | tr ',' ' '); do | |
| value=$(az keyvault secret show \ | |
| --name $secretName \ | |
| --vault-name $VAULT_NAME \ | |
| --query value \ | |
| --output tsv) | |
| echo "::add-mask::$value" | |
| echo "$secretName=$value" >> $GITHUB_OUTPUT | |
| echo "$secretName=$value" >> $GITHUB_ENV | |
| done | |
| # Authenticate with GCP Workload Identity Pool | |
| # Exports GCP ENV Vars: | |
| # - GCP_PROJECT | |
| # - GOOGLE_APPLICATION_CREDENTIALS | |
| - id: 'auth' | |
| if: matrix.require-gcp-credentials == 'true' | |
| name: 'Authenticate to Google Cloud' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_NAME }} | |
| service_account: ${{ secrets.GCP_WIF_SA_EMAIL }} | |
| create_credentials_file: true | |
| export_environment_variables: true | |
| cleanup_credentials: true | |
| # Download the required certificates into files, and set env var pointing to their names | |
| - name: Setup certs | |
| if: matrix.required-certs != '' | |
| run: | | |
| for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do | |
| CERT_FILE=$(mktemp --suffix .pfx) | |
| echo "Downloading cert $CERT_NAME into file $CERT_FILE" | |
| rm $CERT_FILE && \ | |
| az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE | |
| echo 'Setting $CERT_NAME to' "$CERT_FILE" | |
| echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV | |
| done | |
| - name: Setup Terraform | |
| if: matrix.require-terraform == 'true' | |
| uses: hashicorp/[email protected] | |
| - name: Set Cloudflare env vars | |
| if: matrix.require-cloudflare-credentials == 'true' | |
| run: | | |
| echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV | |
| echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV | |
| - name: Set AWS env vars | |
| if: matrix.require-aws-credentials == 'true' | |
| run: | | |
| echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV | |
| echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV | |
| - name: Configure AWS Credentials | |
| if: matrix.require-aws-credentials == 'true' | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} | |
| aws-region: us-west-1 | |
| - name: Start MongoDB | |
| if: matrix.mongodb-version != '' | |
| uses: supercharge/[email protected] | |
| with: | |
| mongodb-version: ${{ matrix.mongodb-version }} | |
| mongodb-replica-set: test-rs | |
| - name: Set up Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version-file: 'go.mod' | |
| - name: Install Node.js ${{ matrix.nodejs-version }} | |
| if: matrix.nodejs-version != '' | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: ${{ matrix.nodejs-version }} | |
| - name: Start KinD | |
| uses: helm/[email protected] | |
| if: matrix.require-kind == 'true' | |
| - name: Download Go dependencies | |
| run: | | |
| go mod download | |
| go install github.com/axw/gocov/gocov@${{ env.GOCOV_VER }} | |
| go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VER }} | |
| - name: Run setup script | |
| if: matrix.setup-script != '' | |
| run: .github/scripts/components-scripts/${{ matrix.setup-script }} | |
| - name: Catch setup failures | |
| if: failure() | |
| run: | | |
| echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV | |
| - name: Run tests | |
| continue-on-error: true | |
| run: | | |
| set -e | |
| KIND=$(echo ${{ matrix.component }} | cut -d. -f1) | |
| NAME=$(echo ${{ matrix.component }} | cut -d. -f2-) | |
| KIND_UPPER="$(tr '[:lower:]' '[:upper:]' <<< ${KIND:0:1})${KIND:1}" | |
| if [ "${KIND}" = "secretstores" ]; then | |
| KIND_UPPER=SecretStore | |
| fi | |
| echo "Running tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... " | |
| echo "Source Pacakge: " ${{ matrix.source-pkg }} | |
| set +e | |
| gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json \ | |
| --junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.xml --format standard-verbose -- \ | |
| -p 2 -count=1 -timeout=15m -tags=conftests ./tests/conformance --run="Test${KIND_UPPER}Conformance/${NAME}" -coverprofile=cover.out \ | |
| -covermode=set -coverpkg=${{ matrix.source-pkg }} | |
| status=$? | |
| echo "Completed tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... " | |
| if test $status -ne 0; then | |
| echo "Setting CONFORMANCE_FAILURE" | |
| echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV | |
| fi | |
| set -e | |
| # Fail the step if we found no test to run | |
| if grep -q "warning: no tests to run" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json ; then | |
| echo "::error:: No test was found for component ${{ matrix.component }}" | |
| exit -1 | |
| fi | |
| - name: Delete downloaded up certs | |
| if: always() && matrix.required-certs != '' | |
| run: | | |
| for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do | |
| CERT_FILE=$(printenv $CERT_NAME) | |
| echo "Cleaning up the certificate file $CERT_FILE..." | |
| rm $CERT_FILE || true | |
| done | |
| - name: Check conformance test passed | |
| continue-on-error: false | |
| run: | | |
| echo "CONFORMANCE_FAILURE=$CONFORMANCE_FAILURE" | |
| if [[ -v CONFORMANCE_FAILURE ]]; then | |
| exit 1 | |
| fi | |
| - name: Prepare test result info | |
| if: always() | |
| run: | | |
| mkdir -p tmp/result_files | |
| echo "Writing to tmp/result_files/${{ matrix.component }}.txt" | |
| if [[ "${{ env.CONFORMANCE_FAILURE }}" == "true" ]]; then | |
| echo "0" >> "tmp/result_files/${{ matrix.component }}.txt" | |
| else | |
| echo "1" >> "tmp/result_files/${{ matrix.component }}.txt" | |
| fi | |
| - name: Upload result files | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: result_files | |
| path: tmp/result_files | |
| retention-days: 1 | |
| - name: Prepare coverage report file to upload | |
| if: github.event_name == 'schedule' | |
| run: | | |
| mkdir -p tmp/conf_code_cov | |
| cp cover.out tmp/conf_code_cov/${{ env.SOURCE_PATH_LINEAR }}.out | |
| - name: Upload coverage report file | |
| uses: actions/upload-artifact@v3 | |
| if: github.event_name == 'schedule' | |
| with: | |
| name: conf_code_cov | |
| path: tmp/conf_code_cov | |
| retention-days: 7 | |
| # Upload logs for test analytics to consume | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@master | |
| with: | |
| name: ${{ matrix.component }}_conformance_test | |
| path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.* | |
| - name: Run destroy script | |
| if: always() && matrix.destroy-script != '' | |
| run: .github/scripts/components-scripts/${{ matrix.destroy-script }} | |
| post_job: | |
| name: Post-completion | |
| runs-on: ubuntu-22.04 | |
| if: always() | |
| needs: | |
| - conformance | |
| - generate-matrix | |
| steps: | |
| - name: Parse repository_dispatch payload | |
| if: github.event_name == 'repository_dispatch' | |
| working-directory: ${{ github.workspace }} | |
| run: | | |
| if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then | |
| echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV | |
| echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV | |
| fi | |
| - name: Download test result artifact | |
| if: always() && env.PR_NUMBER != '' | |
| uses: actions/download-artifact@v3 | |
| continue-on-error: true | |
| id: testresults | |
| with: | |
| name: result_files | |
| path: tmp/result_files | |
| - name: Build message | |
| if: always() && env.PR_NUMBER != '' | |
| # Abusing of the github-script action to be able to write this in JS | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}') | |
| const basePath = '${{ steps.testresults.outputs.download-path }}' | |
| const testType = 'conformance' | |
| const fs = require('fs') | |
| const path = require('path') | |
| let message = `# Components ${testType} test | |
| 🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})** | |
| Commit ref: ${{ env.CHECKOUT_REF }}` | |
| let allSuccess = true | |
| let allFound = true | |
| let notSuccess = [] | |
| let notFound = [] | |
| for (let i = 0; i < allComponents.length; i++) { | |
| let component = allComponents[i] | |
| if (!component) { | |
| continue | |
| } | |
| if (typeof component == 'object') { | |
| component = component.component | |
| } | |
| let found = false | |
| let success = false | |
| try { | |
| let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8') | |
| read = read.split('\n')[0] | |
| switch (read) { | |
| case '1': | |
| found = true | |
| success = true | |
| break | |
| case '0': | |
| found = true | |
| success = false | |
| } | |
| } catch (e) { | |
| // ignore errors, leave found = false | |
| } | |
| if (!found) { | |
| allFound = false | |
| notFound.push(component) | |
| } | |
| if (!success) { | |
| allSuccess = false | |
| notSuccess.push(component) | |
| } | |
| } | |
| if (allSuccess) { | |
| if (allFound) { | |
| message += '\n\n' + `# ✅ All ${testType} tests passed | |
| All tests have reported a successful status` + '\n\n' | |
| } else { | |
| message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status | |
| Although there were no failures reported, some tests did not report a status:` + '\n\n' | |
| for (let i = 0; i < notFound.length; i++) { | |
| message += '- ' + notFound[i] + '\n' | |
| } | |
| message += '\n' | |
| } | |
| } else { | |
| message += '\n\n' + `# ❌ Some ${testType} tests failed | |
| These tests failed:` + '\n\n' | |
| for (let i = 0; i < notSuccess.length; i++) { | |
| message += '- ' + notSuccess[i] + '\n' | |
| } | |
| message += '\n' | |
| if (!allFound) { | |
| message += 'Additionally, some tests did not report a status:\n\n' | |
| for (let i = 0; i < notFound.length; i++) { | |
| message += '- ' + notFound[i] + '\n' | |
| } | |
| message += '\n' | |
| } | |
| } | |
| fs.writeFileSync('message.txt', message) | |
| - name: Replace PR comment | |
| if: env.PR_NUMBER != '' | |
| uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba | |
| with: | |
| header: ${{ github.run_id }} | |
| number: ${{ env.PR_NUMBER }} | |
| GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }} | |
| path: message.txt |