System Test #443
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: System Test | |
| on: | |
| schedule: | |
| # Run a test every monday 6 am | |
| - cron: '0 6 * * 1' | |
| workflow_dispatch: | |
| inputs: | |
| releaseUrl: | |
| description: The URL to the release file. An empty URL performs a build of 'main' first. | |
| required: false | |
| localBuildRepo: | |
| description: The Git repository used when a local build is required (defaults to drogue-iot/drogue-cloud). | |
| required: false | |
| localBuildRef: | |
| description: The Git ref (tag/branch/sha) used when a local build is required (defaults to main). | |
| required: false | |
| containerRegistry: | |
| description: The container registry prefix | |
| required: false | |
| default: ghcr.io/drogue-iot | |
| cluster: | |
| description: The cluster type | |
| default: kind | |
| required: true | |
| serverType: | |
| description: The server type to use (this value has a price attached!). | |
| default: ccx22 | |
| required: true | |
| prNr: | |
| description: A PR number to build and report back to | |
| required: false | |
| installerArgs: | |
| description: Additional installer arguments (like `-s key=value`) | |
| required: false | |
| delayCleanup: | |
| description: "This will delay the cleanup until the maximum job timeout exceeds. (NOTE: costs more money!)" | |
| required: false | |
| env: | |
| HCLOUD_VERSION: "1.30.3" | |
| VERSION_HELM: "v3.10.1" | |
| jobs: | |
| init: | |
| name: Initialize | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| cluster: ${{ steps.clusterType.outputs.cluster }} | |
| buildRepo: ${{ steps.localBuild.outputs.buildRepo }} | |
| buildRef: ${{ steps.localBuild.outputs.buildRef }} | |
| needBuild: ${{ steps.needBuild.outputs.needBuild }} | |
| buildReportId: ${{ steps.build.outputs.reportId }} | |
| buildReportUrl: ${{ steps.build.outputs.reportUrl }} | |
| steps: | |
| - id: clusterType | |
| name: Evaluate cluster type | |
| env: | |
| CLUSTER: ${{ github.event.inputs.cluster }} | |
| run: | | |
| echo "cluster=${CLUSTER:-kind}" >> $GITHUB_OUTPUT | |
| - id: needBuild | |
| name: Evaluate if we need a local build | |
| run: | | |
| echo "Release URL: ${{ github.event.inputs.releaseUrl }}" | |
| echo "needBuild=${{ github.event.inputs.releaseUrl == '' }}" >> $GITHUB_OUTPUT | |
| - id: localBuild | |
| name: Evaluate local build information | |
| env: | |
| GIT_REPO: ${{ github.event.inputs.localBuildRepo }} | |
| GIT_REF: ${{ github.event.inputs.localBuildRef }} | |
| run: | | |
| echo "buildRepo=${GIT_REPO:-https://github.com/drogue-iot/drogue-cloud}" >> $GITHUB_OUTPUT | |
| echo "buildRef=${GIT_REF:-main}" >> $GITHUB_OUTPUT | |
| - id: build | |
| name: Generate the build timestamp and some values with it | |
| run: | | |
| Y=$(date "+%Y") | |
| M=$(date "+%m") | |
| D=$(date "+%d") | |
| echo "name=year=${Y}" >> $GITHUB_OUTPUT | |
| echo "name=month=${M}" >> $GITHUB_OUTPUT | |
| echo "name=day=${D}" >> $GITHUB_OUTPUT | |
| echo "reportId=${Y}-${M}-${D}-test-run-${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT | |
| echo "reportUrl=https://drogue-iot.github.io/drogue-cloud-testing/test-report/${Y}/${M}/${D}/test-run-${GITHUB_RUN_ID}.html" >> $GITHUB_OUTPUT | |
| create-runner: | |
| name: Create runner | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install hcloud CLI | |
| run: | | |
| mkdir -p download | |
| cd download | |
| curl -sSL https://github.com/hetznercloud/cli/releases/download/v${HCLOUD_VERSION}/hcloud-linux-amd64.tar.gz -o hcloud.tar.gz | |
| tar xzf hcloud.tar.gz | |
| install hcloud /usr/local/bin | |
| - name: Check hcloud binary | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| run: hcloud version | |
| - name: Create runner | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.API_PAT }} | |
| SERVER_TYPE: ${{ github.event.inputs.serverType }} | |
| run: | | |
| RUNNER_URL=$(gh api repos/drogue-iot/drogue-cloud-testing/actions/runners/downloads --jq '.[] | select(.os=="linux" and .architecture=="x64") | .download_url') | |
| TOKEN=$(gh api -X POST repos/${{ github.repository }}/actions/runners/registration-token --template '{{ .token }}') | |
| echo ::add-mask::$TOKEN | |
| sed "s/@@TOKEN@@/${TOKEN}/g" hcloud/cloud-init-ubuntu.yaml | \ | |
| sed "s|@@RUNNER_URL@@|${RUNNER_URL}|g" | \ | |
| sed 's/@@RUNNER@@/${{github.run_id}}/g' > init.yaml | |
| cat init.yaml | |
| hcloud server create --name testing-runner-${{github.run_id}} --datacenter hel1-dc2 --image ubuntu-22.04 --type "${SERVER_TYPE:-ccx22}" --ssh-key 3746242 --user-data-from-file init.yaml | |
| - name: Waiting for runner to become ready | |
| id: runner | |
| timeout-minutes: 10 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.API_PAT }} | |
| run: | | |
| ID=$(gh api repos/${{ github.repository }}/actions/runners --jq '.runners[]? | select(.name=="testing-runner-${{github.run_id}}") | .id') | |
| while [[ -z "$ID" ]]; do | |
| sleep 10 | |
| ID=$(gh api repos/${{ github.repository }}/actions/runners --jq '.runners[]? | select(.name=="testing-runner-${{github.run_id}}") | .id') | |
| done | |
| echo "id=${ID}" >> $GITHUB_OUTPUT | |
| test: | |
| name: Run tests | |
| runs-on: [ "self-hosted", "hetzner" ] | |
| # we set an overall timeout for this job, to prevent the machine from running indefinitely (must be greater than | |
| # the step timeout below) | |
| timeout-minutes: 240 # 4h | |
| needs: | |
| - create-runner | |
| - init | |
| env: | |
| CLUSTER: ${{ needs.init.outputs.cluster }} | |
| DEBIAN_FRONTEND: noninteractive | |
| steps: | |
| - run: | | |
| echo "Cluster type: ${{ needs.init.outputs.cluster }}" | |
| echo "::notice title=Cluster type::${{ needs.init.outputs.cluster }}" | |
| echo "Require local build: ${{ needs.init.outputs.needBuild }}" | |
| echo "::notice title=Test subject::${{ ( (needs.init.outputs.needBuild == 'true') && 'Local build' ) || format('Released artifact: {0}', github.event.inputs.releaseUrl) }}" | |
| - run: | | |
| echo "REPORT_PAGE_ID=${{ needs.init.outputs.buildReportId }}" >> $GITHUB_ENV | |
| echo "REPORT_PAGE_URL=${{ needs.init.outputs.buildReportUrl }}" >> $GITHUB_ENV | |
| - name: Add .cargo/bin to path | |
| run: | | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 # required to switch branch later | |
| - uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| - run: env | sort | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get -y install gcc libssl-dev make httpie xvfb | |
| - name: Add .local/bin to path | |
| run: | | |
| mkdir -p "$HOME/.local/bin" | |
| echo "$HOME/.local/bin" >> $GITHUB_PATH | |
| - name: Install drg | |
| if: ${{ false }} | |
| run: | | |
| mkdir -p download | |
| cd download | |
| curl -sL https://github.com/drogue-iot/drg/releases/download/v0.11.0/drg-0.11.0-linux-amd64.tar.gz -o drg.tar.gz | |
| tar --strip-components=1 -xvzf drg.tar.gz | |
| mv drg "$HOME/.local/bin" | |
| - name: Build drg | |
| run: | | |
| cargo install drg --version 0.11.0 --force | |
| - name: Check drg version | |
| run: drg --version | |
| - name: Install geckodriver | |
| run: | | |
| mkdir -p download | |
| cd download | |
| curl -sL https://github.com/mozilla/geckodriver/releases/download/v0.32.0/geckodriver-v0.32.0-linux64.tar.gz -o geckodriver.tar.gz | |
| tar -xzvf geckodriver.tar.gz | |
| mv geckodriver "$HOME/.local/bin" | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v3 | |
| with: | |
| version: ${{ env.VERSION_HELM }} | |
| - name: Install kubectl | |
| run: | | |
| curl -sL https://dl.k8s.io/release/v1.25.3/bin/linux/amd64/kubectl -o "$HOME/.local/bin/kubectl" | |
| chmod a+x "$HOME/.local/bin/kubectl" | |
| - name: Install Minikube | |
| if: ${{ needs.init.outputs.cluster == 'minikube' }} | |
| run: | | |
| curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 | |
| sudo install minikube-linux-amd64 /usr/bin/minikube | |
| - name: Install markdown-test-report | |
| run: | | |
| curl -sL https://github.com/ctron/markdown-test-report/releases/download/v0.3.4/markdown-test-report-linux-amd64 -o "$HOME/.local/bin/markdown-test-report" | |
| chmod a+x "$HOME/.local/bin/markdown-test-report" | |
| # get the content to test | |
| - name: Download release | |
| if: ${{ needs.init.outputs.needBuild == 'false' }} | |
| run: | | |
| mkdir release | |
| cd release | |
| curl -sL "${{ github.event.inputs.releaseUrl }}" -o release.zip | |
| unzip release.zip | |
| mv drogue-install-*/* . | |
| find . | |
| - name: Checkout for local build | |
| if: ${{ needs.init.outputs.needBuild == 'true' }} | |
| env: | |
| GH_TOKEN: ${{ secrets.API_PAT }} | |
| run: | | |
| git version | |
| if [ -z "${{ github.event.inputs.prNr }}" ]; then | |
| git clone ${{ needs.init.outputs.buildRepo }} --branch ${{ needs.init.outputs.buildRef }} | |
| cd drogue-cloud | |
| else | |
| git clone ${{ needs.init.outputs.buildRepo }} | |
| cd drogue-cloud | |
| gh pr checkout "${{ github.event.inputs.prNr }}" | |
| fi | |
| git log -1 | |
| git config --file=.gitmodules submodule."helm-charts".url https://github.com/drogue-iot/drogue-cloud-helm-charts.git | |
| git submodule sync | |
| git submodule update --init | |
| cd deploy/helm | |
| git log -1 | |
| - uses: actions/cache@v3 | |
| if: ${{ needs.init.outputs.needBuild == 'true' }} | |
| with: | |
| path: | | |
| drogue-cloud/target/ | |
| key: ${{ runner.os }}-cargo-drogue-${{ hashFiles('**/drogue-cloud/Cargo.lock') }} | |
| - name: Perform a local build | |
| if: ${{ needs.init.outputs.needBuild == 'true' }} | |
| run: | | |
| cd drogue-cloud | |
| make build build-images SKIP_SERVER=1 | |
| make tag-images CONTAINER_REGISTRY=dev.local IMAGE_TAG=${{ github.run_id }} SKIP_SERVER=1 | |
| make -C installer | |
| cd .. | |
| mkdir release | |
| cd release | |
| ls ../drogue-cloud/installer/build | |
| tar --strip-components=1 -xvzf ../drogue-cloud/installer/build/drogue-install-${CLUSTER:-kind}-latest.tar.gz | |
| cd .. | |
| - name: Upload locally built installer | |
| if: ${{ needs.init.outputs.needBuild == 'true' }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: installer | |
| path: | | |
| drogue-cloud/installer/build/drogue-install-*-latest.tar.gz | |
| - name: Check for installer script | |
| run: | | |
| cd release | |
| test -x ./scripts/drgadm | |
| # start the cluster | |
| - name: Create k8s kind Cluster | |
| if: ${{ needs.init.outputs.cluster == 'kind' }} | |
| uses: helm/[email protected] | |
| with: | |
| wait: 300s | |
| cluster_name: kind | |
| config: kind/cluster-config.yaml | |
| - name: Load images in kind | |
| if: ${{ ( needs.init.outputs.needBuild == 'true' ) && ( needs.init.outputs.cluster == 'kind' ) }} | |
| run: make -C drogue-cloud kind-load CONTAINER_REGISTRY=dev.local IMAGE_TAG=${{ github.run_id }} SKIP_SERVER=1 | |
| - name: Create Minikube cluster | |
| if: ${{ needs.init.outputs.cluster == 'minikube' }} | |
| run: | | |
| minikube config set driver kvm2 | |
| # the "start" call should align with what we have in the defaults | |
| minikube start --cpus 4 --memory 14336 --disk-size 20gb --addons ingress | |
| sudo minikube tunnel & | |
| # Start deployment | |
| - name: Deploy Drogue (release build) | |
| if: ${{ needs.init.outputs.needBuild == 'false' }} | |
| env: | |
| DEBUG: "true" | |
| run: | | |
| cd release | |
| ./scripts/drgadm deploy \ | |
| ${{ github.event.inputs.installerArgs }} \ | |
| -s defaults.images.repository=${{ github.event.inputs.containerRegistry }} | |
| timeout-minutes: 20 | |
| - name: Deploy drogue (local build) | |
| if: ${{ needs.init.outputs.needBuild == 'true' }} | |
| env: | |
| TEST_CERTS_IMAGE: "dev.local/test-cert-generator:${{ github.run_id }}" | |
| run: | | |
| cd release | |
| ./scripts/drgadm deploy \ | |
| -m \ | |
| ${{ github.event.inputs.installerArgs }} \ | |
| -s drogueCloudCore.defaults.images.repository=dev.local \ | |
| -S drogueCloudCore.defaults.images.tag=${{ github.run_id }} \ | |
| -s drogueCloudCore.defaults.images.pullPolicy=Never \ | |
| -s drogueCloudExamples.defaults.images.repository=dev.local \ | |
| -S drogueCloudExamples.defaults.images.tag=${{ github.run_id }} \ | |
| -s drogueCloudExamples.defaults.images.pullPolicy=Never | |
| timeout-minutes: 20 | |
| # Run tests | |
| - name: Run tests | |
| # we limit the tests to 3h (must be less than the overall timeout for this job, see above) | |
| timeout-minutes: 180 # 3h | |
| run: xvfb-run --auto-servernum make -s test RUST_LOG=debug TEST_ARGS="-Z unstable-options --format json --report-time" CLUSTER=${{ needs.init.outputs.cluster }} CERT_BASE="release/build/certs" > test-output.json | |
| # using tee combined with the xvfb-run swallows up the error response, even though "pipefail" is enabled | |
| - run: cat test-output.json | |
| if: ${{ always() }} | |
| - name: Render test report | |
| if: ${{ always() }} | |
| run: | | |
| if [ "${{ needs.init.outputs.needBuild }}" == "true" ]; then | |
| markdown-test-report --git drogue-cloud | |
| else | |
| markdown-test-report | |
| fi | |
| - name: Render PR comment | |
| if: ${{ always() && ( github.event.inputs.prNr != '' ) && ( needs.init.outputs.needBuild == 'true' ) }} | |
| env: | |
| GH_TOKEN: ${{ secrets.API_PAT }} | |
| run: | | |
| if markdown-test-report --git drogue-cloud --no-front-matter --summary --output - > "comment.md"; then | |
| echo "**Report:** ${REPORT_PAGE_URL}" >> "comment.md" | |
| else | |
| echo "Test run finished, but did not create a test report: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" > "comment.md" | |
| fi | |
| gh pr comment --repo drogue-iot/drogue-cloud "${{ github.event.inputs.prNr }}" -F "comment.md" | |
| - name: Upload raw test results | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-output | |
| path: | | |
| test-output.json | |
| test-output.md | |
| comment.md | |
| - name: Revert changes | |
| if: ${{ always() }} | |
| run: | | |
| git status | |
| git diff | |
| # seems like this file has changed, try restoring | |
| git restore README.md | |
| - name: Collect logs | |
| if: ${{ always() }} | |
| run: | | |
| mkdir logs | |
| sudo journalctl > logs/journal.log | |
| kubectl get ns > logs/namespaces.log | |
| for ns in $(kubectl get ns --no-headers -o=custom-columns=:metadata.name); do | |
| mkdir -p "logs/$ns" | |
| ./.github/scripts/collect_logs.sh "logs/$ns" "$ns" | |
| done | |
| cp geckodriver.log logs/ | |
| - name: Package logs | |
| if: ${{ always() }} | |
| run: tar czvf logs.tar.gz logs | |
| - name: Upload logs | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: logs | |
| path: logs.tar.gz | |
| - name: Upload screenshots | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: screenshots | |
| path: screenshots | |
| - name: Delay cleanup | |
| if: ${{ github.event.inputs.delayCleanup }} | |
| run: sleep infinity | |
| destroy-runner: | |
| name: Destroy runner | |
| needs: | |
| - create-runner | |
| - test | |
| runs-on: ubuntu-22.04 | |
| if: ${{ always() }} # the runner must always be deleted | |
| steps: | |
| - name: Install hcloud CLI | |
| run: | | |
| mkdir -p download | |
| cd download | |
| curl -sSL https://github.com/hetznercloud/cli/releases/download/v${HCLOUD_VERSION}/hcloud-linux-amd64.tar.gz -o hcloud.tar.gz | |
| tar xzf hcloud.tar.gz | |
| install hcloud /usr/local/bin | |
| - name: Check hcloud binary | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| run: hcloud version | |
| - name: Destroy runner | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| run: hcloud server delete testing-runner-${GITHUB_RUN_ID} | |
| - name: Remove runner | |
| if: ${{ always() }} # always remove | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.API_PAT }} | |
| # Although we have the runner ID in the create step, it may be that there is a race condition between | |
| # failing to detect the creation and destroying. So we try to find the runner now anyway. | |
| run: | | |
| for id in $(gh api repos/${{ github.repository }}/actions/runners --jq '.runners[] | select(.name=="testing-runner-${{github.run_id}}") | .id'); do | |
| echo "ID: $id" | |
| gh api -X DELETE repos/${{ github.repository }}/actions/runners/$id | |
| done | |
| publish-results: | |
| name: Publish results | |
| needs: | |
| - init | |
| - test | |
| runs-on: ubuntu-22.04 | |
| if: ${{ always() }} # results should always get published | |
| steps: | |
| - run: | | |
| echo "REPORT_PAGE_ID=${{ needs.init.outputs.buildReportId }}" >> $GITHUB_ENV | |
| echo "REPORT_PAGE_URL=${{ needs.init.outputs.buildReportUrl }}" >> $GITHUB_ENV | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: gh-pages | |
| path: repo | |
| - name: Fetch test output | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test-output | |
| path: test-output | |
| - name: List output | |
| run: find test-output | |
| - name: Append PR comment | |
| if: ${{ github.event.inputs.prNr != '' && needs.init.outputs.needBuild == 'true' }} | |
| env: | |
| GH_TOKEN: ${{ secrets.API_PAT }} | |
| run: | | |
| gh pr comment --repo drogue-iot/drogue-cloud "${{ github.event.inputs.prNr }}" -F "test-output/comment.md" | |
| - name: Publish test results | |
| # TODO: publish test results in a different repository | |
| run: | | |
| cd repo | |
| git config --global user.name "GitHub workflow" | |
| git config --global user.email '[email protected]' | |
| cp ../test-output/test-output.md _posts/${REPORT_PAGE_ID}.md | |
| git add -A -- _posts | |
| if ! git diff --cached --exit-code -- _posts; then | |
| echo "Changes have been detected, commit and push ..." | |
| git commit -m "Automatic update ($GITHUB_RUN_ID/$GITHUB_RUN_NUMBER)" | |
| git log --graph --abbrev-commit --date=relative -n 5 | |
| git push | |
| echo "::notice title=Test Report::${REPORT_PAGE_URL}" | |
| else | |
| echo "No changes have been detected since last build, nothing to publish" | |
| fi |