Stable Components Certification Tests #499

Workflow file for this run

.github/workflows/certification.yml at 33a5fd2

	# ------------------------------------------------------------
	# Copyright 2021 The Dapr Authors
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	# http://www.apache.org/licenses/LICENSE-2.0
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	# ------------------------------------------------------------

	name: Stable Components Certification Tests

	on:
	repository_dispatch:
	types: [certification-test]
	workflow_dispatch:
	schedule:
	- cron: '25 /8 * *'
	push:
	branches:
	- 'release-*'
	pull_request:
	branches:
	- 'master'
	- 'release-*'

	jobs:
	# Based on whether this is a PR or a scheduled run, we will run a different
	# subset of the certification tests. This allows all the tests not requiring
	# secrets to be executed on pull requests.
	generate-matrix:
	runs-on: ubuntu-22.04
	steps:
	- name: Parse repository_dispatch payload
	if: github.event_name == 'repository_dispatch'
	working-directory: ${{ github.workspace }}
	run: \|
	if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
	echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
	echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
	fi

	- name: Check out code
	uses: actions/checkout@v3
	with:
	repository: ${{ env.CHECKOUT_REPO }}
	ref: ${{ env.CHECKOUT_REF }}

	- name: Generate test matrix
	id: generate-matrix
	env:
	VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
	run: \|
	if [ -z "$VAULT_NAME" ]; then
	# Do not include cloud tests when credentials are not available
	node .github/scripts/test-info.mjs certification false
	else
	# Include cloud tests
	node .github/scripts/test-info.mjs certification true
	fi

	- name: Create PR comment
	if: env.PR_NUMBER != ''
	uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
	with:
	header: ${{ github.run_id }}
	number: ${{ env.PR_NUMBER }}
	GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
	message: \|
	# Components certification test

	🔗 [Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

	Commit ref: ${{ env.CHECKOUT_REF }}

	outputs:
	test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }}

	certification:
	name: ${{ matrix.component }} certification

	# Add "id-token" with the intended permissions.
	# Needed by the 'Authenticate to Google Cloud' step.
	permissions:
	contents: 'read'
	id-token: 'write'

	runs-on: ubuntu-22.04
	env:
	UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}}
	GOCOV_VER: "v1.1.0"
	GOTESTSUM_VER: "v1.9.0"
	defaults:
	run:
	shell: bash

	needs:
	- generate-matrix

	strategy:
	fail-fast: false # Keep running even if one component fails
	matrix:
	include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }}

	steps:
	- name: Set default payload repo and ref
	run: \|
	echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
	echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV

	- name: Parse repository_dispatch payload
	if: github.event_name == 'repository_dispatch'
	run: \|
	if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
	echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV
	echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
	fi

	- name: Check out code
	uses: actions/checkout@v3
	with:
	repository: ${{ env.CHECKOUT_REPO }}
	ref: ${{ env.CHECKOUT_REF }}

	- name: Configure environment
	run: \|
	# Output file
	echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV

	# Certification test and source path
	TEST_COMPONENT=$(echo "${{ matrix.component }}" \| sed -E 's/\./\//g')
	echo "TEST_PATH=tests/certification/${TEST_COMPONENT}" >> $GITHUB_ENV
	SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}"
	echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV
	# converts slashes to dots in this string, so that it doesn't consider them sub-folders
	SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" \|sed 's#/#\.#g')
	echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV

	# Current time (used by Terraform)
	echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV}

	- uses: Azure/login@v1
	with:
	creds: ${{ secrets.AZURE_CREDENTIALS }}
	if: matrix.required-secrets != ''

	# Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal:
	# az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID
	# Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated
	- name: Setup secrets
	if: matrix.required-secrets != ''
	env:
	VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
	run: \|
	secrets="${{ matrix.required-secrets }}"
	for secretName in $(echo -n $secrets \| tr ',' ' '); do
	value=$(az keyvault secret show \
	--name $secretName \
	--vault-name $VAULT_NAME \
	--query value \
	--output tsv)
	echo "::add-mask::$value"
	echo "$secretName=$value" >> $GITHUB_OUTPUT
	echo "$secretName=$value" >> $GITHUB_ENV
	done

	# Authenticate with GCP Workload Identity Pool
	# Exports GCP ENV Vars:
	# - GCP_PROJECT
	# - GOOGLE_APPLICATION_CREDENTIALS
	- id: 'auth'
	if: matrix.require-gcp-credentials == 'true'
	name: 'Authenticate to Google Cloud'
	uses: 'google-github-actions/auth@v1'
	with:
	token_format: 'access_token'
	workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_NAME }}
	service_account: ${{ secrets.GCP_WIF_SA_EMAIL }}
	create_credentials_file: true
	export_environment_variables: true
	cleanup_credentials: true

	# Download the required certificates into files, and set env var pointing to their names
	- name: Setup certs
	if: matrix.required-certs != ''
	working-directory: ${{ env.TEST_PATH }}
	run: \|
	for CERT_NAME in $(echo "${{ matrix.required-certs }}" \| sed 's/,/ /g'); do
	CERT_FILE=$(mktemp --suffix .pfx)
	echo "Downloading cert $CERT_NAME into file $CERT_FILE"
	rm $CERT_FILE && \
	az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE
	echo 'Setting $CERT_NAME to' "$CERT_FILE"
	echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV
	done

	- name: Setup Terraform
	uses: hashicorp/[email protected]
	if: matrix.require-terraform == 'true'

	- name: Set Cloudflare env vars
	if: matrix.require-cloudflare-credentials == 'true'
	run: \|
	echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV
	echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV

	- name: Set AWS env vars
	if: matrix.require-aws-credentials == 'true'
	run: \|
	echo "AWS_REGION=us-west-1" >> $GITHUB_ENV
	echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV
	echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV

	- name: Configure AWS Credentials
	uses: aws-actions/configure-aws-credentials@v1
	if: matrix.require-aws-credentials == 'true'
	with:
	aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY }}"
	aws-secret-access-key: "${{ secrets.AWS_SECRET_KEY }}"
	aws-region: "${{ env.AWS_REGION }}"

	- name: Set up Go
	id: setup-go
	uses: actions/setup-go@v3
	with:
	go-version-file: 'go.mod'

	- name: Download Go dependencies
	working-directory: ${{ env.TEST_PATH }}
	run: \|
	go mod download
	go install github.com/axw/gocov/gocov@${{ env.GOCOV_VER }}
	go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VER }}

	- name: Run setup script
	if: matrix.setup-script != ''
	run: .github/scripts/components-scripts/${{ matrix.setup-script }}

	- name: Catch setup failures
	if: failure()
	run: \|
	echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV

	- name: Run tests
	continue-on-error: false
	working-directory: ${{ env.TEST_PATH }}
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
	AWS_REGION: "${{ env.AWS_REGION }}"
	run: \|
	echo "Running certification tests for ${{ matrix.component }} ... "
	echo "Source Pacakge: " ${{ matrix.source-pkg }}
	export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore
	set +e
	gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \
	--junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \
	-coverprofile=cover.out -covermode=set -tags=certtests -timeout=30m -coverpkg=${{ matrix.source-pkg }}
	status=$?
	echo "Completed certification tests for ${{ matrix.component }} ... "
	if test $status -ne 0; then
	echo "Setting CERTIFICATION_FAILURE"
	export CERTIFICATION_FAILURE=true
	fi
	set -e
	mkdir -p tmp/cert_code_cov_files
	cat cover.out >> tmp/cert_code_cov_files/${{ env.SOURCE_PATH_LINEAR }}.out
	cat tmp/cert_code_cov_files/${{ env.SOURCE_PATH_LINEAR }}.out

	# Fail the step if we found no test to run
	if grep -q "\[no test files\]" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json ; then
	echo "::error:: No certification test file was found for component ${{ matrix.component }}"
	exit -1
	fi

	for CERT_NAME in $(echo "${{ matrix.required-certs }}" \| sed 's/,/ /g'); do
	CERT_FILE=$(printenv $CERT_NAME)

	echo "Cleaning up the certificate file $CERT_FILE..."
	rm $CERT_FILE \|\| true
	done

	if [[ -v CERTIFICATION_FAILURE ]]; then
	echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV
	exit 1
	else
	echo "CERTIFICATION_FAILURE=false" >> $GITHUB_ENV
	fi

	- name: Upload Cert Coverage Report File
	uses: actions/upload-artifact@v3
	if: github.event_name == 'schedule'
	with:
	name: cert_code_cov
	path: ${{ env.TEST_PATH }}/tmp/cert_code_cov_files
	retention-days: 7

	- name: Prepare test result info
	if: always()
	run: \|
	mkdir -p tmp/result_files
	echo "Writing to tmp/result_files/${{ matrix.component }}.txt"
	if [[ "${{ env.CERTIFICATION_FAILURE }}" == "true" ]]; then
	echo "0" >> "tmp/result_files/${{ matrix.component }}.txt"
	else
	echo "1" >> "tmp/result_files/${{ matrix.component }}.txt"
	fi

	- name: Upload result files
	uses: actions/upload-artifact@v3
	if: always()
	with:
	name: result_files
	path: tmp/result_files
	retention-days: 1

	# Upload logs for test analytics to consume
	- name: Upload test results
	if: always()
	uses: actions/upload-artifact@master
	with:
	name: ${{ matrix.component }}_certification_test
	path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.*

	- name: Run destroy script
	if: always() && matrix.destroy-script != ''
	run: .github/scripts/components-scripts/${{ matrix.destroy-script }}

	post_job:
	name: Post-completion
	runs-on: ubuntu-22.04
	if: always()
	needs:
	- certification
	- generate-matrix
	steps:
	- name: Parse repository_dispatch payload
	if: github.event_name == 'repository_dispatch'
	working-directory: ${{ github.workspace }}
	run: \|
	if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
	echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
	echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
	fi

	- name: Download test result artifact
	if: always() && env.PR_NUMBER != ''
	uses: actions/download-artifact@v3
	continue-on-error: true
	id: testresults
	with:
	name: result_files
	path: tmp/result_files

	- name: Build message
	if: always() && env.PR_NUMBER != ''
	# Abusing of the github-script action to be able to write this in JS
	uses: actions/github-script@v6
	with:
	script: \|
	const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}')
	const basePath = '${{ steps.testresults.outputs.download-path }}'
	const testType = 'certification'

	const fs = require('fs')
	const path = require('path')

	let message = `# Components ${testType} test

	🔗 [Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

	Commit ref: ${{ env.CHECKOUT_REF }}`

	let allSuccess = true
	let allFound = true
	let notSuccess = []
	let notFound = []
	for (let i = 0; i < allComponents.length; i++) {
	let component = allComponents[i]
	if (!component) {
	continue
	}
	if (typeof component == 'object') {
	component = component.component
	}
	let found = false
	let success = false
	try {
	let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8')
	read = read.split('\n')[0]
	switch (read) {
	case '1':
	found = true
	success = true
	break
	case '0':
	found = true
	success = false
	}
	} catch (e) {
	// ignore errors, leave found = false
	}

	if (!found) {
	allFound = false
	notFound.push(component)
	}
	if (!success) {
	allSuccess = false
	notSuccess.push(component)
	}
	}

	if (allSuccess) {
	if (allFound) {
	message += '\n\n' + `# ✅ All ${testType} tests passed

	All tests have reported a successful status` + '\n\n'
	} else {
	message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status

	Although there were no failures reported, some tests did not report a status:` + '\n\n'
	for (let i = 0; i < notFound.length; i++) {
	message += '- ' + notFound[i] + '\n'
	}
	message += '\n'
	}
	} else {
	message += '\n\n' + `# ❌ Some ${testType} tests failed

	These tests failed:` + '\n\n'
	for (let i = 0; i < notSuccess.length; i++) {
	message += '- ' + notSuccess[i] + '\n'
	}
	message += '\n'

	if (!allFound) {
	message += 'Additionally, some tests did not report a status:\n\n'
	for (let i = 0; i < notFound.length; i++) {
	message += '- ' + notFound[i] + '\n'
	}
	message += '\n'
	}
	}

	fs.writeFileSync('message.txt', message)

	- name: Replace PR comment
	if: always() && env.PR_NUMBER != ''
	uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
	with:
	header: ${{ github.run_id }}
	number: ${{ env.PR_NUMBER }}
	GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
	path: message.txt

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Stable Components Certification Tests #499

Workflow file

Stable Components Certification Tests #499

Jobs

Run details

Workflow file for this run