From 2f895cd36f3638eadaa576d63cbf032e4d7e01d3 Mon Sep 17 00:00:00 2001 From: lilyLuLiu Date: Wed, 20 Nov 2024 09:17:44 +0800 Subject: [PATCH] [QE] update github action for linux arm64 tests --- .github/workflows/linux-qe-template.yml | 111 +++++++++++++++++++----- .github/workflows/linux-qe-test.yml | 9 +- 2 files changed, 89 insertions(+), 31 deletions(-) diff --git a/.github/workflows/linux-qe-template.yml b/.github/workflows/linux-qe-template.yml index 9b75717b97..9869ca20ec 100644 --- a/.github/workflows/linux-qe-template.yml +++ b/.github/workflows/linux-qe-template.yml @@ -16,11 +16,14 @@ on: jobs: linux-qe: - runs-on: [self-hosted, linux, testing-farm] + runs-on: ubuntu-24.04 permissions: statuses: write # needed to update commit status (pending/failure/sucess) checks: write # as documented in https://github.com/mikepenz/action-junit-report?tab=readme-ov-file#pr-run-permissions steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Download gh context id: download-gh-context-artifact uses: actions/download-artifact@v4 @@ -31,7 +34,10 @@ jobs: - name: prepare run: | - sudo yum install podman openssh-server -y + # Install Testing farm CLI + pip3 install --user tft-cli + sudo apt install podman openssh-server -y + testing-farm --help # Get origin commit sha for testing commit_sha=$(cat gh_context.json | jq -r '.event.after') @@ -46,10 +52,13 @@ jobs: if [[ "${{inputs.qe-type}}" == "e2e" ]]; then status_context="${status_context}-${{inputs.preset}}" fi - status_context="${status_context}/windows-${{matrix.windows-version}}-${{matrix.windows-featurepack}}" + status_context="${status_context}/Linux-ARM64" echo "status_context=${status_context}" >> "$GITHUB_ENV" - echo "${PULL_SECRET}" > pull-secret + version_line=$(cat Makefile | grep "OPENSHIFT_VERSION ?=") + bundle_version=${version_line##*= } + echo "bundle_version=${bundle_version}" >> "$GITHUB_ENV" + - name: Download linux binary uses: actions/download-artifact@v4 @@ -86,10 +95,14 @@ jobs: TESTING_FARM_API_TOKEN: ${{ secrets.TESTING_FARM_API_TOKEN }} PULL_SECRET: ${{ secrets.PULL_SECRET }} run: | + echo "${PULL_SECRET}" > pull-secret + # the target can only be accessed through a bastion (which can only be accessed from self-hosted runner) # as so we need to map the ssh-agent from the host to the containers used to access the target host - rm -f id_rsa id_rsa.pub - ssh-keygen -t rsa -N '' -f id_rsa -q + + #rm -f id_rsa id_rsa.pub + ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa -q + cp ~/.ssh/id_rsa . eval $(ssh-agent -s) echo $SSH_AUTH_SOCK > ssh_auth_sock echo $SSH_AGENT_PID > ssh_agent_pid @@ -97,7 +110,8 @@ jobs: # reserve machine from testing farm export TESTING_FARM_API_TOKEN=${TESTING_FARM_API_TOKEN} - testing-farm reserve --compose Fedora-40 --duration 240 --arch aarch64 --hardware memory='>= 12 GB' --hardware virtualization.is-supported='true' --ssh-public-key id_rsa.pub --no-autoconnect | tee info + testing-farm reserve --compose Fedora-40 --duration 480 --arch aarch64 --hardware memory='>= 16 GB' --hardware virtualization.is-supported='true' --no-autoconnect | tee info1 + sed -r 's/\x1B\[[0-9;]*[mK]//g' info1 > info machine=`tail -n 1 info` echo ${machine##*@} > host echo crctest > username @@ -107,15 +121,20 @@ jobs: echo ${request:1} > requestid # Create a non-root user for testing on the reserved machine - ${machine:1} 'useradd crctest' < /dev/null - ${machine:1} 'echo "crctest:redhat" | chpasswd' < /dev/null - ${machine:1} 'usermod -aG wheel crctest' < /dev/null - ${machine:1} 'echo "crctest ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/crctest-users' < /dev/null - ${machine:1} 'mkdir -p /home/crctest/.ssh' < /dev/null - ${machine:1} 'cp /root/.ssh/authorized_keys /home/crctest/.ssh/' < /dev/null - ${machine:1} 'chown crctest /home/crctest/.ssh/authorized_keys' < /dev/null + ssh_cmd="ssh -o StrictHostKeyChecking=no ${machine##*ssh}" + ssh-keyscan -H testing-farm.io >> ~/.ssh/known_hosts + echo ${machine##*ssh} + echo $ssh_cmd + $ssh_cmd 'useradd crctest' < /dev/null + $ssh_cmd 'echo "crctest:redhat" | chpasswd' < /dev/null + $ssh_cmd 'usermod -aG wheel crctest' < /dev/null + $ssh_cmd 'echo "crctest ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/crctest-users' < /dev/null + $ssh_cmd 'mkdir -p /home/crctest/.ssh' < /dev/null + $ssh_cmd 'cp /root/.ssh/authorized_keys /home/crctest/.ssh/' < /dev/null + $ssh_cmd 'chown -R crctest:crctest /home/crctest/.ssh/' < /dev/null # Install CRC on the reserved machine + echo "Start installing crc on reserved machine" podman run --rm -d --privileged --name crc-linux-install-${{inputs.qe-type}}-${{inputs.preset}} \ -e TARGET_HOST=$(cat host) \ -e TARGET_HOST_USERNAME=$(cat username) \ @@ -131,25 +150,65 @@ jobs: -v ${PWD}:/data:z \ -v ${PWD}/crc:/opt/crc-support/crc:z \ quay.io/crc-org/ci-crc-support:v2.0.0-dev-linux crc-support/run.sh \ - -targetPath "/root/crc-support" \ + -targetPath "/home/crctest/crc-support" \ -install 'true' \ -aName 'crc' \ -freshEnv 'false' \ -download 'false' podman logs -f crc-linux-install-${{inputs.qe-type}}-${{inputs.preset}} + # Download arm64 bundle in the reserved machine + echo "Start download bundle on reserved machine" + + if [[ "${{inputs.preset}}" == "microshift" ]]; then + bundle_url="https://crcqe-asia.s3.ap-south-1.amazonaws.com/bundles/microshift/${{ env.bundle_version }}-arm64" + bundle_name="crc_microshift_libvirt_${{ env.bundle_version }}_arm64.crcbundle" + else + bundle_url="https://crcqe-asia.s3.ap-south-1.amazonaws.com/bundles/openshift/${{ env.bundle_version }}-arm64" + bundle_name="crc_libvirt_${{ env.bundle_version }}_arm64.crcbundle" + fi + podman run --rm -d --privileged --name download_bundle \ + -e TARGET_HOST=$(cat host) \ + -e TARGET_HOST_USERNAME=$(cat username) \ + -e TARGET_HOST_KEY_PATH=/data/id_rsa \ + -e BASTION_HOST_USERNAME=$(cat bastion_username) \ + -e BASTION_HOST=$(cat bastion_host) \ + -e TARGET_FOLDER=crc-support \ + -e TARGET_CLEANUP='false' \ + -e OUTPUT_FOLDER=/data \ + -e DEBUG='true' \ + -e SSH_AUTH_SOCK=$(cat ssh_auth_sock) \ + -v "$(cat ssh_auth_sock):$(cat ssh_auth_sock)" \ + -v ${PWD}:/data:z \ + quay.io/crc-org/ci-crc-support:v2.0.0-dev-linux crc-support/run.sh \ + -targetPath "/home/crctest" \ + -install 'false' \ + -aBaseURL $bundle_url \ + -aName $bundle_name \ + -freshEnv 'false' \ + -download 'true' + podman logs -f download_bundle + # load image podman load -i crc-${{inputs.qe-type}}-linux-arm64.tar + $ssh_cmd 'chmod +x /usr/local/bin/crc' < /dev/null # run CRC test - cmd="crc-qe/run.sh -junitFilename crc-${{inputs.qe-type}}-junit.xml -targetFolder crc-qe" + cmd="crc-qe/run.sh -bundleLocation /home/crctest/$bundle_name -junitFilename crc-${{inputs.qe-type}}-junit.xml -targetFolder crc-qe" if [[ "${{inputs.qe-type}}" == "e2e" ]]; then if [[ "${{inputs.preset}}" == "microshift" ]]; then cmd="${cmd} -e2eTagExpression '@story_microshift'" else - cmd="${cmd} -e2eTagExpression '~@minimal && ~@story_microshift'" + cmd="${cmd} -e2eTagExpression '~@minimal && ~@story_microshift && ~@cert_rotation'" + fi + else + if [[ "${{inputs.preset}}" == "microshift" ]]; then + cmd="${cmd} -labelFilter 'microshift-preset'" + else + cmd="${cmd} -labelFilter 'openshift-preset'" fi fi + echo "Start running test on reserved machine" podman run --rm -d --privileged --name crc-${{inputs.qe-type}}-${{inputs.preset}} \ -e TARGET_HOST=$(cat host) \ -e TARGET_HOST_USERNAME=$(cat username) \ @@ -189,6 +248,17 @@ jobs: **/*.results **/*.log + - name: Return machine and clear env + env: + TESTING_FARM_API_TOKEN: ${{ secrets.TESTING_FARM_API_TOKEN }} + if: always() + run: | + export TESTING_FARM_API_TOKEN=${TESTING_FARM_API_TOKEN} + testing-farm cancel $(cat requestid) + podman rmi quay.io/crcont/crc-${{inputs.qe-type}}:gh-linux-arm64 + rm -r results + kill $(cat ssh_agent_pid) + - name: Update status of the PR check if: always() run: | @@ -208,12 +278,7 @@ jobs: https://api.github.com/repos/${{ github.repository }}/statuses/${{ env.commit_sha }} \ -d "${data}" - - name: Return machine and clear env - env: - TESTING_FARM_API_TOKEN: ${{ secrets.TESTING_FARM_API_TOKEN }} - run: | - export TESTING_FARM_API_TOKEN=${TESTING_FARM_API_TOKEN} - testing-farm cancel $(cat requestid) + diff --git a/.github/workflows/linux-qe-test.yml b/.github/workflows/linux-qe-test.yml index 4d67a466c4..93ec5856af 100644 --- a/.github/workflows/linux-qe-test.yml +++ b/.github/workflows/linux-qe-test.yml @@ -17,14 +17,7 @@ jobs: fail-fast: false matrix: qe-type: ['e2e','integration'] - preset: ['openshift', 'microshift', 'all'] - exclude: - - qe-type: 'e2e' - preset: 'all' - - qe-type: 'integration' - preset: 'openshift' - - qe-type: 'integration' - preset: 'microshift' + preset: ['openshift', 'microshift'] with: trigger-workflow-run-id: ${{ github.event.workflow_run.id }} qe-type: ${{matrix.qe-type}}