chore(ci): build python binding with self hosted runner (#13225)

.github/actions/build_bindings_python/action.yml

@@ -24,32 +24,24 @@ runs:
       id: toolchain
       shell: bash
       run: |
-        bash ./scripts/setup/dev_setup.sh -yb
         RUST_TOOLCHAIN=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' rust-toolchain.toml)
         echo "RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" >> $GITHUB_OUTPUT
 
-    # NOTE: for exporting ACTIONS_RUNTIME_TOKEN and ACTIONS_CACHE_URL
-    - name: Expose GitHub Runtime
-      uses: crazy-max/ghaction-github-runtime@v2
-      if: env.RUNNER_PROVIDER == 'github'
-
     - name: Get opts
       id: opts
       shell: bash
       run: |
-        echo "DOCKER_OPTS=--env RUSTC_WRAPPER=sccache --env SCCACHE_GHA_ENABLED=true" >> $GITHUB_OUTPUT
-        if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+        if [[ -z "${{ inputs.version }}" ]]; then
           echo "BUILD_ARGS=--strip --out dist" >> $GITHUB_OUTPUT
-          echo "BUILD_PROFILE=debug" >> $GITHUB_ENV
         else
           echo "BUILD_ARGS=--release --strip --out dist" >> $GITHUB_OUTPUT
-          echo "BUILD_PROFILE=release" >> $GITHUB_ENV
         fi
 
     - name: Cross setup for macOS
       if: endsWith(inputs.target, '-darwin')
       shell: bash
       run: |
+        bash ./scripts/setup/dev_setup.sh -yb
         echo "JEMALLOC_SYS_WITH_LG_PAGE=14" >> $GITHUB_ENV
         echo "JEMALLOC_SYS_WITH_MALLOC_CONF=oversize_threshold:0,dirty_decay_ms:5000,muzzy_decay_ms:5000" >> $GITHUB_ENV
 
@@ -63,7 +55,7 @@ runs:
         # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
         rustup-components: rust-std rustfmt
         args: ${{ steps.opts.outputs.BUILD_ARGS }}
-        docker-options: ${{ steps.opts.outputs.DOCKER_OPTS }}
+        docker-options: --env RUSTC_WRAPPER=sccache --env SCCACHE_GCS_RW_MODE=READ_WRITE --env SCCACHE_GCS_BUCKET=databend-ci --env SCCACHE_GCS_KEY_PREFIX=cache/sccache/
         before-script-linux: ../../scripts/setup/dev_setup.sh -yb
 
     - name: Run tests
@@ -74,9 +66,3 @@ runs:
         pip install dist/*.whl
         pip install pytest pyarrow pandas polars
         pytest -v tests/*
-
-    - name: Upload artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: dist
-        path: src/bendpy/dist/*.whl

.github/workflows/bindings.python.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - "src/**"
+      - ".github/workflows/bindings.python.yml"
   workflow_call:
     inputs:
       tag:
@@ -17,44 +18,36 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
-env:
-  RUNNER_PROVIDER: github
-
 jobs:
-  build_linux:
-    name: build-${{ matrix.target }}
-    runs-on: ubuntu-latest
+  linux:
+    name: ${{ matrix.target }}
+    runs-on: [self-hosted, X64, Linux, 8c16g, gcp]
     strategy:
       matrix:
         target:
           - x86_64-unknown-linux-gnu
     steps:
-      - name: Free Disk Space (Ubuntu)
-        uses: jlumbroso/free-disk-space@main
-        with:
-          # this might remove tools that are actually needed,
-          # if set to "true" but frees about 6 GB
-          tool-cache: false
-          # all of these default to true, but feel free to set to
-          # "false" if necessary for your workflow
-          android: true
-          dotnet: true
-          haskell: true
-          large-packages: false
-          docker-images: true
-          swap-storage: true
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ./.github/actions/build_bindings_python
         with:
           target: ${{ matrix.target }}
           version: ${{ inputs.tag }}
+      - name: Publish to PyPI
+        if: inputs.tag
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }}
+        uses: PyO3/maturin-action@v1
+        with:
+          command: upload
+          args: --skip-existing *
+          working-directory: src/bendpy
 
-  build_macos:
-    name: build-${{ matrix.target }}
-    if: github.event_name != 'pull_request'
-    runs-on: macos-11
+  macos:
+    if: inputs.tag
+    name: ${{ matrix.target }}
+    runs-on: macos-latest
     strategy:
       matrix:
         target:
@@ -68,18 +61,12 @@ jobs:
         with:
           target: ${{ matrix.target }}
           version: ${{ inputs.tag }}
-
-  release:
-    # publish release only the version endsWith 0
-    # if: endsWith(inputs.tag, '0')
-    if: inputs.tag
-    name: Publish to PyPI
-    needs: [build_linux, build_macos]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/download-artifact@v3
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        if: inputs.tag
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASSWORD }}
+        uses: PyO3/maturin-action@v1
         with:
-          password: ${{ secrets.pypi_password }}
-          skip-existing: true
+          command: upload
+          args: --skip-existing *
+          working-directory: src/bendpy

.github/workflows/dev.yml

@@ -21,6 +21,8 @@ jobs:
       any_src_changed: ${{ steps.src.outputs.any_changed }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Check Source File Changes
         uses: tj-actions/changed-files@v39
         id: src

scripts/setup/dev_setup.sh

@@ -578,8 +578,9 @@ if [[ "$INSTALL_CHECK_TOOLS" == "true" ]]; then
 	if [[ -f scripts/setup/rust-tools.txt ]]; then
 		export RUSTFLAGS="-C target-feature=-crt-static"
 		cargo install cargo-quickinstall
+		cargo quickinstall cargo-binstall
 		while read -r tool; do
-			cargo quickinstall "$tool"
+			cargo binstall "$tool"
 		done <scripts/setup/rust-tools.txt
 	fi