Nightly Builds #317
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Nightly Builds | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # Every night midnight | |
| # This workflow needs to pass if there is a PR changing it. | |
| pull_request: | |
| paths: | |
| - ".github/workflows/nightly.yaml" | |
| workflow_dispatch: # Or manually - for testing | |
| jobs: | |
| build-and-test: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| path: s3gw | |
| submodules: true | |
| - name: Checkout Ceph HEAD | |
| working-directory: s3gw/ceph | |
| run: | | |
| git fetch | |
| git checkout s3gw | |
| git submodule update --init --recursive | |
| - name: Checkout s3tests | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ceph/s3-tests | |
| path: s3tests | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install Dependencies | |
| run: | | |
| YQ_GH_URL=https://github.com/mikefarah/yq/releases/download | |
| YQ_VERSION=v4.31.1 | |
| YQ_CHECKSUM=1aef844cbecbbf81036449ea5e7dfcdf19d2d374fab6303fdb8f849d04275d76 | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| bc \ | |
| wget \ | |
| s3cmd | |
| # Unfortunately, since yq is only available through snap on Ubuntu and | |
| # that doesn't work in docker containers (at least not out of the | |
| # box), this abomination is the way to go to install yq. | |
| echo "${YQ_CHECKSUM} yq" >> checksum | |
| wget -O yq "${YQ_GH_URL}/${YQ_VERSION}/yq_linux_amd64" | |
| sha256sum -c checksum \ | |
| && sudo mv yq /usr/bin/yq \ | |
| && sudo chmod +x /usr/bin/yq | |
| # known to work version of boto3 | |
| python3 -m pip install boto3==1.24.96 | |
| python3 -m pip install -r s3tests/requirements.txt | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Context | |
| run: | | |
| docker context create builder | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| endpoint: builder | |
| - name: Quay Login | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: quay.io | |
| username: ${{ secrets.QUAY_USERNAME }} | |
| password: ${{ secrets.QUAY_PASSWORD }} | |
| - name: Generate Tag based on Date | |
| id: date | |
| run: | | |
| DATE="$(date +%Y-%m-%d)" | |
| echo "tag=nightly-${DATE}" >> $GITHUB_OUTPUT | |
| - name: Build Nightly Container | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| build-args: | | |
| CMAKE_BUILD_TYPE=Release | |
| S3GW_VERSION=${{ steps.date.outputs.tag }} | |
| NPROC=16 | |
| QUAY_EXPIRATION=1w | |
| tags: | | |
| quay.io/s3gw/s3gw:${{ steps.date.outputs.tag }} | |
| quay.io/s3gw/s3gw:nightly-latest | |
| file: s3gw/Dockerfile | |
| context: s3gw | |
| - name: Run S3tests | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.STATUS_PUSH_TOKEN }} | |
| run: | | |
| set +e # don't exit on error | |
| set -x | |
| # needed for GNU parallel because the version in GH runners is bugged | |
| mkdir -p ${HOME}/.parallel | |
| export DEBUG=1 | |
| export S3TEST_PARALLEL=ON | |
| export CEPH_DIR="${GITHUB_WORKSPACE}/s3gw/ceph" | |
| export OUTPUT_DIR="${GITHUB_WORKSPACE}/s3test-results" | |
| export S3GW_CONTAINER="quay.io/s3gw/s3gw:nightly-latest" | |
| export FORCE_CONTAINER=ON | |
| export FORCE_DOCKER=ON | |
| export \ | |
| FIXTURES="${CEPH_DIR}/qa/rgw/store/sfs/tests/fixtures" | |
| export S3TEST_REPO="${GITHUB_WORKSPACE}/s3tests" | |
| export S3TEST_CONF="${FIXTURES}/s3tests.conf" | |
| export S3TEST_LIST="${FIXTURES}/s3-tests.txt" | |
| # There are some s3tests that don't finish at all. Only run | |
| # known-to-pass tests for now. TODO: fix inifite looping tests | |
| # sed -r -i 's/^# //' "${S3TEST_LIST}" | |
| pushd s3tests | |
| ${GITHUB_WORKSPACE}/s3gw/tools/tests/s3tests-runner.sh | |
| popd | |
| git clone \ | |
| https://user:${GITHUB_TOKEN}@github.com/aquarist-labs/s3gw-status | |
| pushd s3gw-status | |
| git config user.name "github nightly bot" | |
| git config user.email "[email protected]" | |
| cp \ | |
| "${OUTPUT_DIR}/report.json" \ | |
| results/s3tests/${{ steps.date.outputs.tag }}.json | |
| git add results/s3tests/${{ steps.date.outputs.tag }}.json | |
| git commit -m "Nightly Results $(date +%Y-%m-%d)" | |
| git push | |
| # We expect some tests to fail, we just want to see _which_ tests fail | |
| # exit 0 |