Update README.md #6010
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python | |
| # This file builds and tests Python binary wheels across major platforms. The | |
| # wheels that are produced are used for PyPi releases. The test step uses a | |
| # fresh job and installs the wheel the ensure it works in isolation. The is also | |
| # a job which produces the source distribution and ensures that can be built to. | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'releases/**' | |
| - rlos2023/test | |
| pull_request: | |
| branches: | |
| - '*' | |
| - rlos2023/test | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| linux-python-build: | |
| name: manylinux.amd64.py${{ matrix.config.version}}.build | |
| container: | |
| image: vowpalwabbit/manylinux2010-build:latest | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| config: | |
| - { version: "3.6", base_path: /opt/python/cp36-cp36m/, include_dir_name: python3.6m/ } | |
| - { version: "3.7", base_path: /opt/python/cp37-cp37m/, include_dir_name: python3.7m/ } | |
| - { version: "3.8", base_path: /opt/python/cp38-cp38/, include_dir_name: python3.8/ } | |
| - { version: "3.9", base_path: /opt/python/cp39-cp39/, include_dir_name: python3.9/ } | |
| - { version: "3.10", base_path: /opt/python/cp310-cp310/, include_dir_name: python3.10/ } | |
| fail-fast: false | |
| steps: | |
| # v1 must be used because newer versions require a node.js version that will not run on this old image. | |
| - uses: actions/checkout@v1 | |
| with: | |
| submodules: recursive | |
| - name: Build wheel | |
| shell: bash | |
| run: | | |
| ${{ matrix.config.base_path }}bin/pip wheel . -w wheel_output/ --global-option --cmake-options="-DSTATIC_LINK_VW_JAVA=On;-DPython_INCLUDE_DIR='${{ matrix.config.base_path }}include/${{ matrix.config.include_dir_name }}'" --verbose | |
| auditwheel repair wheel_output/*whl -w audit_output/ | |
| - name: Upload built wheel | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: manylinux_amd64_${{ matrix.config.version }} | |
| path: audit_output/ | |
| linux-python-test: | |
| name: manylinux.amd64.py${{ matrix.version }}.test | |
| needs: linux-python-build | |
| container: | |
| image: python:${{ matrix.version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| version: ["3.6", "3.7", "3.8", "3.9", "3.10"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v1 | |
| with: | |
| submodules: recursive | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: manylinux_amd64_${{ matrix.version }} | |
| path: built_wheel | |
| - name: Test wheel | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt | |
| pip install pytest vw-executor twine | |
| pip install --force-reinstall built_wheel/*.whl | |
| twine check built_wheel/*.whl | |
| python -m pytest ./python/tests/ | |
| python ./python/tests/run_doctest.py | |
| - name: Run vw tests as Python module | |
| shell: bash | |
| # Onethread is not supported in the Python wrapper so those tests must be skipped | |
| # Stdin is not supported | |
| # Help output tests assume --onthread is in the output | |
| # Fail tests are not included because the python stack trace causes the output to differ | |
| # Daemon tests are skipped | |
| # Tests without datafiles are skipped | |
| run: | | |
| cd test | |
| python run_tests.py -E 0.001 -f --skip_spanning_tree_tests --vw_bin_path "python3 -m vowpalwabbit" --skip_test 60 61 92 96 149 152 177 193 194 195 220 275 276 324 325 326 349 350 356 357 358 385 389 390 391 392 393 400 399 401 403 431 440 | |
| linux-python-sdist-bundle: | |
| name: ubuntu-latest.amd64.py3.8.sdist-bundle | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.8' | |
| architecture: 'x64' | |
| - name: Install dependencies | |
| shell: bash | |
| run: python setup.py sdist | |
| - name: Upload built wheel | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: python_source_distribution | |
| path: dist/*.tar.gz | |
| if-no-files-found: error | |
| linux-python-sdist-build-test: | |
| name: ubuntu-latest.amd64.py3.8.sdist-build-test | |
| needs: linux-python-sdist-bundle | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| architecture: 'x64' | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: python_source_distribution | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| sudo apt update | |
| sudo apt install libboost-test-dev libboost-python-dev zlib1g-dev cmake g++ | |
| - name: Install source dist | |
| shell: bash | |
| run: pip install python_source_distribution/*.tar.gz | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt | |
| pip install pytest vw-executor | |
| - name: Run unit tests | |
| shell: bash | |
| run: | | |
| python -m pytest ./python/tests/ | |
| linux-python-build-aarch64: | |
| name: manylinux.aarch64.py${{ matrix.config.version }}.build | |
| # Aarch builds are slow and are only run on push and not PR | |
| if: ${{ github.event_name == 'push' }} | |
| strategy: | |
| matrix: | |
| config: | |
| - { version: "3.7", base_path: /opt/python/cp37-cp37m/, include_dir_name: python3.7m/ } | |
| - { version: "3.8", base_path: /opt/python/cp38-cp38/, include_dir_name: python3.8/ } | |
| - { version: "3.9", base_path: /opt/python/cp39-cp39/, include_dir_name: python3.9/ } | |
| - { version: "3.10", base_path: /opt/python/cp310-cp310/, include_dir_name: python3.10/ } | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Set up QEMU | |
| id: qemu | |
| uses: docker/setup-qemu-action@v1 | |
| - name: Pull image | |
| run: docker pull vowpalwabbit/manylinux2014_aarch64-build | |
| - name: Build Wheel | |
| run: | | |
| docker run --rm -v ${{ github.workspace }}:/ws:rw --workdir=/ws \ | |
| vowpalwabbit/manylinux2014_aarch64-build \ | |
| bash -exc '${{ matrix.config.base_path }}bin/pip wheel . -w wheel_output/ --global-option --cmake-options="-DSTATIC_LINK_VW_JAVA=On;-DPython_INCLUDE_DIR='${{ matrix.config.base_path }}include/${{ matrix.config.include_dir_name }}'" --verbose && \ | |
| auditwheel repair wheel_output/*whl -w audit_output/' | |
| - name: Upload built wheel | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: manylinux_aarch64_${{ matrix.config.version }} | |
| path: audit_output/ | |
| linux-python-test-aarch64: | |
| name: manylinux.aarch64.py${{ matrix.config.version }}.test | |
| needs: linux-python-build-aarch64 | |
| # Aarch builds are slow and are only run on push and not PR | |
| if: ${{ github.event_name == 'push' }} | |
| strategy: | |
| matrix: | |
| config: | |
| - { version: "3.7" } | |
| - { version: "3.8" } | |
| - { version: "3.9" } | |
| - { version: "3.10" } | |
| fail-fast: false | |
| env: | |
| py: python${{ matrix.config.version }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Set up QEMU | |
| id: qemu | |
| uses: docker/setup-qemu-action@v1 | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: manylinux_aarch64_${{ matrix.config.version }} | |
| path: built_wheel | |
| - name: Test Wheel | |
| run: | | |
| docker run --rm -v ${{ github.workspace }}:/io:rw --workdir=/io \ | |
| arm64v8/ubuntu \ | |
| bash -exc 'apt-get update && \ | |
| apt install software-properties-common -y && \ | |
| add-apt-repository ppa:deadsnakes/ppa -y && \ | |
| DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \ | |
| apt install -y ${{ env.py }} && \ | |
| apt install -y ${{ env.py }}-venv && \ | |
| ${{ env.py }} -m venv .env && \ | |
| source .env/bin/activate && \ | |
| pip install --upgrade pip && \ | |
| pip install -r requirements.txt && \ | |
| pip install pytest vw-executor twine && \ | |
| pip install --force-reinstall built_wheel/*.whl && \ | |
| twine check built_wheel/*.whl && \ | |
| python --version && \ | |
| python -m pytest ./python/tests/ && \ | |
| deactivate' | |
| macos-python-build: | |
| name: macos.amd64.py${{ matrix.config.version }}.build | |
| runs-on: macos-12 | |
| strategy: | |
| matrix: | |
| config: | |
| - { version: 3.6, include_dir_name: python3.6m/} | |
| - { version: 3.7, include_dir_name: python3.7m/} | |
| - { version: 3.8, include_dir_name: python3.8/} | |
| - { version: 3.9, include_dir_name: python3.9/} | |
| - { version: "3.10", include_dir_name: python3.10/} | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| auto-update-conda: true | |
| python-version: ${{ matrix.config.version }} | |
| - name: Build wheel | |
| shell: bash -l {0} | |
| run: | | |
| conda info | |
| conda install python=${{ matrix.config.version }} wheel zlib boost py-boost flatbuffers | |
| pip wheel . -w wheel_output/ --global-option --cmake-options="-DSTATIC_LINK_VW_JAVA=On;-DPython_INCLUDE_DIR=\"$CONDA_PREFIX/include/${{ matrix.config.include_dir_name }}\"" --verbose | |
| - name: Upload built wheel | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: macos_amd64_${{ matrix.config.version }} | |
| path: wheel_output/ | |
| macos-python-test: | |
| name: macos.amd64.py${{ matrix.version }}.test | |
| needs: macos-python-build | |
| runs-on: macos-12 | |
| strategy: | |
| matrix: | |
| version: [3.6, 3.7, 3.8, 3.9, "3.10"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.version }} | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: macos_amd64_${{ matrix.version }} | |
| path: built_wheel | |
| - name: Test wheel | |
| shell: bash | |
| run: | | |
| pip install -r requirements.txt | |
| pip install pytest vw-executor twine | |
| pip install --force-reinstall built_wheel/*.whl | |
| twine check built_wheel/*.whl | |
| python -m pytest ./python/tests/ | |
| python ./python/tests/run_doctest.py | |
| windows-python-build: | |
| name: windows-2019.amd64.py${{ matrix.config.version }}.build | |
| runs-on: windows-2019 | |
| env: | |
| VCPKG_ROOT: ${{ github.workspace }}\\vcpkg | |
| VCPKG_REF: 53bef8994c541b6561884a8395ea35715ece75db | |
| VCPKG_DEFAULT_BINARY_CACHE: ${{ github.workspace }}\vcpkg_binary_cache | |
| strategy: | |
| matrix: | |
| config: | |
| - { version: "3.6", vcpkg_manifest: "python\\vcpkg_default.json", overlay_ports: "python\\vcpkg_python_override_ports\\python36\\"} | |
| - { version: "3.7", vcpkg_manifest: "python\\vcpkg_default.json", overlay_ports: "python\\vcpkg_python_override_ports\\python37\\"} | |
| - { version: "3.8", vcpkg_manifest: "python\\vcpkg_default.json", overlay_ports: "python\\vcpkg_python_override_ports\\python38\\" } | |
| - { version: "3.9", vcpkg_manifest: "python\\vcpkg_default.json", overlay_ports: "python\\vcpkg_python_override_ports\\python39\\" } | |
| - { version: "3.10", vcpkg_manifest: "python\\vcpkg_python310.json", overlay_ports: "" } | |
| fail-fast: false | |
| steps: | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.config.version }} | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| path: ${{github.workspace}}\vowpal_wabbit | |
| - uses: actions/checkout@v3 | |
| with: | |
| path: ${{ env.VCPKG_ROOT }} | |
| repository: 'microsoft/vcpkg' | |
| ref: ${{ env.VCPKG_REF }} | |
| # We need a deep clone so that we can go back in history for version selection. | |
| fetch-depth: '0' | |
| - name: Restore vcpkg and build artifacts | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ${{ env.VCPKG_DEFAULT_BINARY_CACHE }} | |
| key: | | |
| ${{ env.VCPKG_REF }}-${{ matrix.config.version }}-python-win-cache-invalidate-5 | |
| - run: ${{ env.VCPKG_ROOT }}/bootstrap-vcpkg.bat | |
| - name: Build wheel | |
| shell: powershell | |
| run: | | |
| if (![System.IO.Directory]::Exists("${{ env.VCPKG_DEFAULT_BINARY_CACHE }}")) | |
| { | |
| New-Item -ItemType Directory -Force -Path "${{ env.VCPKG_DEFAULT_BINARY_CACHE }}" | |
| } | |
| ls ${{ github.workspace }} | |
| ls ${{ env.VCPKG_DEFAULT_BINARY_CACHE }} | |
| cd ${{github.workspace}}\\vowpal_wabbit | |
| cp ${{ matrix.config.vcpkg_manifest }} vcpkg.json | |
| pip install wheel | |
| pip wheel . -w ${{github.workspace}}\\wheel_output --global-option --vcpkg-root="${{ env.VCPKG_ROOT }}" --global-option --cmake-generator="Visual Studio 16 2019" --global-option --cmake-options="-DVCPKG_OVERLAY_PORTS=${{github.workspace}}\\vowpal_wabbit\\${{ matrix.config.overlay_ports }}" --verbose | |
| - name: Upload built wheel | |
| uses: actions/upload-artifact@v1 | |
| with: | |
| name: windows_amd64_${{ matrix.config.version }} | |
| path: wheel_output | |
| windows-python-test: | |
| name: ${{ matrix.os }}.amd64.py${{ matrix.version }}.test | |
| needs: windows-python-build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| version: ["3.6", "3.7", "3.8", "3.9", "3.10"] | |
| os: ["windows-2019"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.version }} | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: windows_amd64_${{ matrix.version }} | |
| path: built_wheel | |
| - name: Install deps and test wheel | |
| shell: bash | |
| run: | | |
| export wheel_files=(built_wheel/*) | |
| export wheel_file="${wheel_files[0]}" | |
| echo Installing ${wheel_file}... | |
| pip install -r requirements.txt | |
| pip install pytest vw-executor twine | |
| pip install --force-reinstall ${wheel_file} | |
| twine check ${wheel_file} | |
| python -m pytest .\\python\\tests\\ | |
| python .\\python\\tests\\run_doctest.py |