add support connection pool for google cloud storage, `GCS_CLIENT_POO… #648
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| push: | |
| branches: | |
| - master | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.20" | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v3 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - name: Setup musl | |
| id: setup-musl | |
| run: | | |
| sudo apt-get install -y musl-tools musl-dev | |
| - name: Cache golang | |
| id: cache-golang | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/go/pkg/mod | |
| ~/.cache/go-build | |
| key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - name: Install golang dependencies | |
| run: go mod download -x | |
| if: | | |
| steps.cache-golang.outputs.cache-hit != 'true' | |
| - name: Build clickhouse-backup binary | |
| id: make-race | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_20_X64 }} | |
| run: | | |
| make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup | |
| make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips | |
| make build-race build-race-fips config test | |
| - name: Report unittest coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| file: _coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: unittest-${{ matrix.clickhouse }} | |
| # be careful with encrypt with old OpenSSL - https://habr.com/ru/post/535140/ | |
| # openssl enc -base64 -aes-256-cbc -e -in test/integration/credentials.json -out test/integration/credentials.json.enc -md md5 -k ${VAULT_PASSWORD} | |
| - name: Decrypting credentials for Google Cloud Storage | |
| id: secrets | |
| env: | |
| VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }} | |
| run: | | |
| if [[ "" != "${VAULT_PASSWORD}" ]]; then | |
| openssl version | |
| openssl enc -base64 -aes-256-cbc -d -in test/integration/credentials.json.enc -out test/integration/credentials.json -md md5 -k ${VAULT_PASSWORD} | |
| fi | |
| echo "GCS_TESTS=$(if [ -z "${{ secrets.VAULT_PASSWORD }}" ]; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-gcp-credentials | |
| path: | | |
| test/integration/credentials.json | |
| if-no-files-found: error | |
| retention-days: 1 | |
| if: | | |
| steps.secrets.outputs.GCS_TESTS == 'true' | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| build/linux/amd64/clickhouse-backup | |
| build/linux/arm64/clickhouse-backup | |
| build/linux/amd64/clickhouse-backup-fips | |
| build/linux/arm64/clickhouse-backup-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-test-artifacts | |
| path: | | |
| clickhouse-backup/clickhouse-backup-race | |
| clickhouse-backup/clickhouse-backup-race-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| outputs: | |
| GCS_TESTS: ${{ steps.secrets.outputs.GCS_TESTS }} | |
| testflows: | |
| needs: build | |
| name: Testflows | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| clickhouse: | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Install python venv | |
| run: | | |
| set -x | |
| (dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv) | |
| python3 -m venv ~/venv/qa | |
| - name: Cache python | |
| uses: actions/cache@v3 | |
| id: cache-python | |
| with: | |
| path: ~/venv/qa | |
| key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }} | |
| - name: Install python dependencies | |
| run: | | |
| set -x | |
| ~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt | |
| if: | | |
| steps.cache-python.outputs.cache-hit != 'true' | |
| - name: Running TestFlows tests | |
| env: | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_ENDPOINT: ${{ secrets.QA_AWS_ENDPOINT }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_GCS_CRED_JSON: ${{ secrets.QA_GCS_CRED_JSON }} | |
| # don't change it to avoid not working CI/CD | |
| RUN_TESTS: "*" | |
| run: | | |
| set -x | |
| export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup | |
| command -v docker-compose || (apt-get update && apt-get install -y python3-pip && pip3 install -U docker-compose) | |
| docker-compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| source ~/venv/qa/bin/activate | |
| set +e | |
| ~/venv/qa/bin/python3 ./test/testflows/clickhouse_backup/regression.py --debug --only="${RUN_TESTS:-*}" --log ./test/testflows/raw.log | |
| if [[ "0" != "$?" ]]; then | |
| docker-compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml logs clickhouse_backup | |
| exit 1 | |
| fi | |
| set -e | |
| tfs --debug --no-colors transform compact ./test/testflows/raw.log ./test/testflows/compact.log | |
| tfs --debug --no-colors transform nice ./test/testflows/raw.log ./test/testflows/nice.log.txt | |
| tfs --debug --no-colors transform short ./test/testflows/raw.log ./test/testflows/short.log.txt | |
| tfs --debug --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./test/testflows/raw.log - --confidential --copyright "Altinity LTD" --logo ./test/testflows/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./test/testflows/report.html | |
| sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances | |
| - name: Format testflows coverage | |
| run: | | |
| sudo chmod -Rv a+rw test/testflows/_coverage_/ | |
| ls -la test/testflows/_coverage_ | |
| go tool covdata textfmt -i test/testflows/_coverage_/ -o test/testflows/_coverage_/coverage.out | |
| - name: Report testflows coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| file: test/testflows/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: testflows-${{ matrix.clickhouse }} | |
| # todo wait when resolve https://github.com/actions/upload-artifact/issues/270 and uncomment | |
| # - name: Upload testflows logs | |
| # uses: actions/upload-artifact@v3 | |
| # with: | |
| # name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }} | |
| # path: | | |
| # test/testflows/*.log | |
| # test/testflows/*.log.txt | |
| # test/testflows/clickhouse_backup/_instances/**/*.log | |
| # test/testflows/*.html | |
| # retention-days: 7 | |
| test: | |
| needs: build | |
| name: Test | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.20" | |
| clickhouse: | |
| - '1.1.54394' | |
| - '19.17' | |
| - '20.3' | |
| - '20.8' | |
| - '21.3' | |
| - '21.8' | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v3 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - name: Cache golang | |
| id: cache-golang | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/go/pkg/mod | |
| ~/.cache/go-build | |
| key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: build-gcp-credentials | |
| path: ./test/integration/ | |
| if: | | |
| needs.build.outputs.GCS_TESTS == 'true' | |
| - name: Running integration tests | |
| env: | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| # don't change it to avoid broken CI/CD!!! | |
| # RUN_TESTS: "TestFIPS" | |
| # LOG_LEVEL: "debug" | |
| # FTP_DEBUG: "true" | |
| # S3_DEBUG: "true" | |
| CGO_ENABLED: 0 | |
| GCS_TESTS: ${{ needs.build.outputs.GCS_TESTS }} | |
| RUN_ADVANCED_TESTS: 1 | |
| AZURE_TESTS: 1 | |
| # need for FIPS | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| # need for GCP over S3 | |
| QA_GCS_OVER_S3_ACCESS_KEY: ${{ secrets.QA_GCS_OVER_S3_ACCESS_KEY }} | |
| QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }} | |
| QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }} | |
| run: | | |
| set -x | |
| echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}" | |
| echo "GCS_TESTS=${GCS_TESTS}" | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| if [[ "${CLICKHOUSE_VERSION}" =~ 2[2-9]+ ]]; then | |
| export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server | |
| else | |
| export CLICKHOUSE_IMAGE=yandex/clickhouse-server | |
| fi | |
| if [[ "${CLICKHOUSE_VERSION}" == 2* ]]; then | |
| export COMPOSE_FILE=docker-compose_advanced.yml | |
| else | |
| export COMPOSE_FILE=docker-compose.yml | |
| fi | |
| command -v docker-compose || (apt-get update && apt-get install -y python3-pip && pip3 install -U docker-compose) | |
| export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race" | |
| docker-compose -f test/integration/${COMPOSE_FILE} up -d || ( docker-compose -f test/integration/${COMPOSE_FILE} ps -a && docker-compose -f test/integration/${COMPOSE_FILE} logs clickhouse && exit 1 ) | |
| docker-compose -f test/integration/${COMPOSE_FILE} ps -a | |
| go test -timeout 60m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go | |
| - name: Format integration coverage | |
| run: | | |
| sudo chmod -Rv a+rw test/integration/_coverage_/ | |
| ls -la test/integration/_coverage_ | |
| go tool covdata textfmt -i test/integration/_coverage_/ -o test/integration/_coverage_/coverage.out | |
| - name: Report integration coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| file: test/integration/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: integration-${{ matrix.clickhouse }} | |
| coverage: | |
| needs: | |
| - test | |
| - testflows | |
| name: coverage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Coveralls Finished | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| parallel-finished: true | |
| docker: | |
| needs: | |
| - test | |
| - testflows | |
| name: Docker | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: build-artifacts | |
| path: ./build/linux/ | |
| - name: Extract DOCKER_TAG version | |
| id: docker_tag | |
| run: | | |
| DOCKER_TAG=${GITHUB_REF##*/} | |
| export DOCKER_TAG=${DOCKER_TAG##*\\} | |
| echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Building docker image | |
| env: | |
| DOCKER_REPO: ${{ secrets.DOCKER_REPO }} | |
| DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_USER: ${{ secrets.DOCKER_USER }} | |
| DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} | |
| DOCKER_TAG: ${{ steps.docker_tag.outputs.docker_tag }} | |
| run: | | |
| if [[ "${DOCKER_TOKEN}" != "" ]]; then | |
| export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} | |
| echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY} | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --target=image_short --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --target=image_fips --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --target=image_full --pull --push . | |
| fi | |
| cleanup: | |
| name: Cleanup | |
| needs: | |
| - docker | |
| - coverage | |
| runs-on: ubuntu-22.04 | |
| if: always() | |
| steps: | |
| - name: delete build-artifacts | |
| uses: geekyeggo/delete-artifact@v2 | |
| with: | |
| name: build-artifacts | |
| failOnError: false | |
| - name: delete build-gcp-credentials | |
| uses: geekyeggo/delete-artifact@v2 | |
| with: | |
| name: build-gcp-credentials | |
| failOnError: false |