Reusing Batch Iterators #10627
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| permissions: read-all | |
| on: | |
| push: | |
| branches: [master] | |
| tags: | |
| - v[0-9]+.[0-9]+.[0-9]+** # Tag filters not as strict due to different regex system on Github Actions | |
| paths-ignore: | |
| - 'build-image/**' | |
| - '.github/workflows/build-image.yml' | |
| pull_request: | |
| paths-ignore: | |
| - 'build-image/**' | |
| - '.github/workflows/build-image.yml' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: quay.io/cortexproject/build-image:master-0ddced051 | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Setup Git safe.directory | |
| run: | | |
| echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." | |
| echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" | |
| git config --system --add safe.directory $GITHUB_WORKSPACE | |
| # Commands in the Makefile are hardcoded with an assumed file structure of the CI container | |
| # Symlink ensures paths specified in previous commands don’t break | |
| - name: Sym Link Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/cortexproject/cortex | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex | |
| - name: Lint | |
| run: make BUILD_IN_CONTAINER=false lint | |
| - name: Check Vendor Directory | |
| run: make BUILD_IN_CONTAINER=false mod-check | |
| - name: Check Protos | |
| run: make BUILD_IN_CONTAINER=false check-protos | |
| - name: Check Generated Documentation | |
| run: make BUILD_IN_CONTAINER=false check-doc | |
| - name: Check White Noise. | |
| run: make BUILD_IN_CONTAINER=false check-white-noise | |
| test: | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: quay.io/cortexproject/build-image:master-0ddced051 | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Setup Git safe.directory | |
| run: | | |
| echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." | |
| echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" | |
| git config --system --add safe.directory $GITHUB_WORKSPACE | |
| - name: Sym Link Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/cortexproject/cortex | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex | |
| - name: Run Tests | |
| run: make BUILD_IN_CONTAINER=false test | |
| security: | |
| name: CodeQL | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| # Initializes the CodeQL tools for scanning. | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 | |
| with: | |
| languages: go | |
| - name: Autobuild | |
| uses: github/codeql-action/autobuild@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@aa578102511db1f4524ed59b8cc2bae4f6e88195 # v3.27.6 | |
| build: | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: quay.io/cortexproject/build-image:master-0ddced051 | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Setup Git safe.directory | |
| run: | | |
| echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." | |
| echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" | |
| git config --system --add safe.directory $GITHUB_WORKSPACE | |
| - name: Install Docker Client | |
| run: ./.github/workflows/scripts/install-docker.sh | |
| - name: Sym Link Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/cortexproject/cortex | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex | |
| - name: Build Image | |
| run: | | |
| touch build-image/.uptodate | |
| make BUILD_IN_CONTAINER=false | |
| - name: Build Website | |
| run: | | |
| touch build-image/.uptodate | |
| make BUILD_IN_CONTAINER=false web-build | |
| - name: Upload Website Artifact | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: website public | |
| path: website/public/ | |
| - name: Save Images | |
| run: | | |
| mkdir /tmp/images | |
| ln -s /tmp/images ./docker-images | |
| make BUILD_IN_CONTAINER=false save-images | |
| - name: Create Docker Images Archive | |
| run: tar -cvf images.tar /tmp/images | |
| - name: Upload Docker Images Artifact | |
| uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 | |
| with: | |
| name: Docker Images | |
| path: ./images.tar | |
| integration: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tags: | |
| - requires_docker | |
| - integration_alertmanager | |
| - integration_backward_compatibility | |
| - integration_memberlist | |
| - integration_querier | |
| - integration_ruler | |
| - integration_query_fuzz | |
| steps: | |
| - name: Upgrade golang | |
| uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 | |
| with: | |
| go-version: 1.23.2 | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Install Docker Client | |
| run: sudo ./.github/workflows/scripts/install-docker.sh | |
| - name: Sym Link Expected Path to Workspace | |
| run: | | |
| sudo mkdir -p /go/src/github.com/cortexproject/cortex | |
| sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex | |
| - name: Download Docker Images Artifacts | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images Archive | |
| run: tar -xvf images.tar -C / | |
| - name: Load Docker Images | |
| run: | | |
| ln -s /tmp/images ./docker-images | |
| make BUILD_IN_CONTAINER=false load-images | |
| - name: Preload Images | |
| # We download docker images used by integration tests so that all images are available | |
| # locally and the download time doesn't account in the test execution time, which is subject | |
| # to a timeout | |
| run: | | |
| docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z | |
| docker pull consul:1.8.4 | |
| docker pull gcr.io/etcd-development/etcd:v3.4.7 | |
| if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then | |
| docker pull quay.io/cortexproject/cortex:v1.13.1 | |
| docker pull quay.io/cortexproject/cortex:v1.13.2 | |
| docker pull quay.io/cortexproject/cortex:v1.14.0 | |
| docker pull quay.io/cortexproject/cortex:v1.14.1 | |
| docker pull quay.io/cortexproject/cortex:v1.15.0 | |
| docker pull quay.io/cortexproject/cortex:v1.15.1 | |
| docker pull quay.io/cortexproject/cortex:v1.15.2 | |
| docker pull quay.io/cortexproject/cortex:v1.15.3 | |
| docker pull quay.io/cortexproject/cortex:v1.16.0 | |
| docker pull quay.io/cortexproject/cortex:v1.16.1 | |
| docker pull quay.io/cortexproject/cortex:v1.17.0 | |
| docker pull quay.io/cortexproject/cortex:v1.17.1 | |
| docker pull quay.io/cortexproject/cortex:v1.18.0 | |
| docker pull quay.io/cortexproject/cortex:v1.18.1 | |
| elif [ "$TEST_TAGS" = "integration_query_fuzz" ]; then | |
| docker pull quay.io/cortexproject/cortex:v1.18.1 | |
| docker pull quay.io/prometheus/prometheus:v2.51.0 | |
| docker pull quay.io/prometheus/prometheus:v2.55.1 | |
| fi | |
| docker pull memcached:1.6.1 | |
| docker pull redis:7.0.4-alpine | |
| env: | |
| TEST_TAGS: ${{ matrix.tags }} | |
| - name: Integration Tests | |
| run: | | |
| export CORTEX_IMAGE_PREFIX="${IMAGE_PREFIX:-quay.io/cortexproject/}" | |
| export IMAGE_TAG=$(make image-tag) | |
| export CORTEX_IMAGE="${CORTEX_IMAGE_PREFIX}cortex:$IMAGE_TAG-amd64" | |
| export CORTEX_CHECKOUT_DIR="/go/src/github.com/cortexproject/cortex" | |
| echo "Running integration tests with image: $CORTEX_IMAGE" | |
| go test -tags=integration,${{ matrix.tags }} -timeout 2400s -v -count=1 ./integration/... | |
| env: | |
| IMAGE_PREFIX: ${{ secrets.IMAGE_PREFIX }} | |
| integration-configs-db: | |
| needs: build | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Install Docker Client | |
| run: sudo ./.github/workflows/scripts/install-docker.sh | |
| - name: Download Docker Images Artifact | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images Archive | |
| run: tar -xvf images.tar -C / | |
| - name: Run Integration Configs Tests | |
| # Github Actions does not support TTY in their default runners yet | |
| run: | | |
| touch build-image/.uptodate | |
| MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations | |
| make BUILD_IMAGE=quay.io/cortexproject/build-image:master-0ddced051 TTY='' configs-integration-test | |
| deploy_website: | |
| needs: [build, test] | |
| if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: quay.io/cortexproject/build-image:master-0ddced051 | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| # web-deploy script expects repo to be cloned with ssh for some commands to work | |
| ssh-key: ${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }} | |
| - name: Setup Git safe.directory | |
| run: | | |
| echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." | |
| echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" | |
| git config --system --add safe.directory $GITHUB_WORKSPACE | |
| - name: Sym Link Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/cortexproject/cortex | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex | |
| - name: Download Website Artifact | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: website public | |
| path: website/public | |
| - name: Setup SSH Keys and known_hosts for Github Authentication to Deploy Website | |
| run: | | |
| mkdir -p ~/.ssh | |
| ssh-keyscan github.com >> ~/.ssh/known_hosts | |
| ssh-agent -a $SSH_AUTH_SOCK > /dev/null | |
| ssh-add - <<< "${{ secrets.WEBSITE_DEPLOY_SSH_PRIVATE_KEY }}" | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
| shell: bash | |
| - name: Deploy Website | |
| # SSH is used to authentricate with Github because web-deploy script uses git to checkout and push to gh-pages | |
| run: make BUILD_IN_CONTAINER=false web-deploy | |
| env: | |
| SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
| GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no" | |
| deploy: | |
| needs: [build, test, lint, integration, integration-configs-db] | |
| if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' | |
| runs-on: ubuntu-20.04 | |
| container: | |
| image: quay.io/cortexproject/build-image:master-0ddced051 | |
| steps: | |
| - name: Checkout Repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Setup Git safe.directory | |
| run: | | |
| echo "this step is needed because when running in container, actions/checkout does not set safe.directory effectively." | |
| echo "See https://github.com/actions/runner/issues/2033. We should use --system instead of --global" | |
| git config --system --add safe.directory $GITHUB_WORKSPACE | |
| - name: Install Docker Client | |
| run: ./.github/workflows/scripts/install-docker.sh | |
| - name: Sym link Expected Path to Workspace | |
| run: | | |
| mkdir -p /go/src/github.com/cortexproject/cortex | |
| ln -s $GITHUB_WORKSPACE/* /go/src/github.com/cortexproject/cortex | |
| - name: Download Docker Images Artifact | |
| uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 | |
| with: | |
| name: Docker Images | |
| - name: Extract Docker Images Archive | |
| run: tar -xvf images.tar -C / | |
| - name: Load Images | |
| run: | | |
| ln -s /tmp/images ./docker-images | |
| make BUILD_IN_CONTAINER=false load-images | |
| - name: Deploy | |
| run: | | |
| if [ -n "$DOCKER_REGISTRY_PASSWORD" ]; then | |
| docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD" | |
| fi | |
| if [ -n "$QUAY_REGISTRY_PASSWORD" ]; then | |
| docker login -u "$QUAY_REGISTRY_USER" -p "$QUAY_REGISTRY_PASSWORD" quay.io; | |
| fi | |
| export IMAGE_TAG=$(make image-tag) | |
| ./push-images $NOQUAY | |
| env: | |
| DOCKER_REGISTRY_USER: ${{secrets.DOCKER_REGISTRY_USER}} | |
| DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| QUAY_REGISTRY_USER: ${{secrets.QUAY_REGISTRY_USER}} | |
| QUAY_REGISTRY_PASSWORD: ${{secrets.QUAY_REGISTRY_PASSWORD}} | |
| NOQUAY: ${{secrets.NOQUAY}} |