From ff2731bc7e9039faec9166a048403c57fecb6627 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Fri, 12 Jul 2024 17:21:33 -0400 Subject: [PATCH] Switch to use self-hosted runner for build_wheels action (#31866) * Switch to use self-hosted runner * split os and runner string --- .github/workflows/build_wheels.yml | 52 +++++++++--------------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 5142b0b22c30..dc14359aa887 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -45,7 +45,7 @@ jobs: check_env_variables: timeout-minutes: 5 name: "Check environment variables" - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] env: EVENT_NAME: ${{ github.event_name }} PY_VERSIONS_FULL: "cp38-* cp39-* cp310-* cp311-* cp312-*" @@ -59,8 +59,8 @@ jobs: run: "./scripts/ci/ci_check_are_gcp_variables_set.sh" id: check_gcp_variables env: - GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }} - GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_SA_EMAIL: "not used by self hosted runner" + GCP_SA_KEY: "not used by self hosted runner" GCP_PYTHON_WHEELS_BUCKET: ${{ secrets.GCP_PYTHON_WHEELS_BUCKET }} GCP_PROJECT_ID: "not-needed-here" GCP_REGION: "not-needed-here" @@ -80,7 +80,7 @@ jobs: echo "py-versions-full=$PY_VERSIONS_FULL" >> $GITHUB_OUTPUT build_source: - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] name: Build python source distribution outputs: is_rc: ${{ steps.is_rc.outputs.is_rc }} @@ -190,14 +190,9 @@ jobs: needs: - build_source - check_env_variables - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' steps: - - name: Authenticate on GCP - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Remove existing files on GCS bucket run: gsutil rm -r ${{ env.GCP_PATH }} || true @@ -206,7 +201,7 @@ jobs: needs: - prepare_gcs - check_env_variables - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] if: needs.check_env_variables.outputs.gcp-variables-set == 'true' steps: - name: Download compressed sources from artifacts @@ -215,11 +210,6 @@ jobs: with: name: source_zip path: source/ - - name: Authenticate on GCP - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Copy sources to GCS bucket run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }} @@ -230,19 +220,19 @@ jobs: - build_source env: CIBW_ARCHS_LINUX: ${{matrix.arch}} - runs-on: ${{ matrix.os_python.os }} + runs-on: ${{ matrix.os_python.runner }} strategy: matrix: os_python: [ - {"os": "ubuntu-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-full }}" }, + {"os": "ubuntu-20.04", "runner": [self-hosted, ubuntu-20.04, main], "python": "${{ needs.check_env_variables.outputs.py-versions-full }}" }, # Temporarily pin to macos-13 because macos-latest breaks this build # TODO(https://github.com/apache/beam/issues/31114) - {"os": "macos-13", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" }, - {"os": "windows-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" }, + {"os": "macos-13", "runner": "macos-13", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" }, + {"os": "windows-latest", "runner": "windows-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" }, ] arch: [auto] include: - - os_python: {"os": "ubuntu-latest", "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" } + - os_python: {"os": "ubuntu-20.04", "runner": [self-hosted, ubuntu-20.04, main], "python": "${{ needs.check_env_variables.outputs.py-versions-test }}" } arch: aarch64 steps: - name: Download python source distribution from artifacts @@ -324,16 +314,16 @@ jobs: needs: - build_wheels - check_env_variables - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' strategy: matrix: # Temporarily pin to macos-13 because macos-latest breaks this build # TODO(https://github.com/apache/beam/issues/31114) - os : [ubuntu-latest, macos-13, windows-latest] + os : [ubuntu-20.04, macos-13, windows-latest] arch: [auto] include: - - os: "ubuntu-latest" + - os: ubuntu-20.04 arch: aarch64 steps: - name: Download wheels from artifacts @@ -342,11 +332,6 @@ jobs: with: name: wheelhouse-${{ matrix.os }}${{ (matrix.arch == 'aarch64' && '-aarch64') || '' }} path: wheelhouse/ - - name: Authenticate on GCP - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Copy wheels to GCS bucket run: gsutil cp -r -a public-read wheelhouse/* ${{ env.GCP_PATH }} - name: Create github action information file on GCS bucket @@ -375,14 +360,9 @@ jobs: needs: - upload_wheels_to_gcs - check_env_variables - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' steps: - - name: Authenticate on GCP - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_email: ${{ secrets.GCP_SA_EMAIL }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - name: List file on Google Cloud Storage Bucket run: gsutil ls "${{ env.GCP_PATH }}*" @@ -393,7 +373,7 @@ jobs: needs: - build_source - build_wheels - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-20.04, main] timeout-minutes: 60 if: github.repository_owner == 'apache' && github.event_name == 'schedule' steps: