From d0706c472c4f235a16288086624cdcf7bd6c13b5 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Sun, 27 Feb 2022 18:07:40 +0000 Subject: [PATCH] Build Linux ARM wheels for Python 3.7 and 3.9 --- .github/actions/multistrap/action.yml | 14 +- .github/workflows/build-and-test.yml | 217 ++++++++++++++---- .gitignore | 1 + Dockerfile.build | 3 +- native_client/BUILD | 15 +- native_client/definitions.mk | 2 +- .../multistrap_raspbian_bullseye.conf | 14 ++ 7 files changed, 214 insertions(+), 52 deletions(-) create mode 100644 native_client/multistrap_raspbian_bullseye.conf diff --git a/.github/actions/multistrap/action.yml b/.github/actions/multistrap/action.yml index c038cd333..74fba64b7 100644 --- a/.github/actions/multistrap/action.yml +++ b/.github/actions/multistrap/action.yml @@ -1,8 +1,8 @@ name: "multistrap install" description: "Install a system root using multistrap" inputs: - arch: - description: "Target arch" + multistrap-config: + description: "Multistrap config file to use" required: true packages: description: "Extra packages to install" @@ -20,13 +20,11 @@ runs: run: | set -xe - multistrap_conf="" - if [ "${{ inputs.arch }}" = "armv7" ]; then - multistrap_conf=multistrap_raspbian_buster.conf + multistrap_conf="${{ inputs.multistrap-config }}" + if [ "$multistrap_conf" = "multistrap_raspbian_buster.conf" ]; then + wget http://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring_20120528.2_all.deb && sudo dpkg -i raspbian-archive-keyring_20120528.2_all.deb + elif [ "$multistrap_conf" = "multistrap_raspbian_bullseye.conf" ]; then wget http://archive.raspbian.org/raspbian/pool/main/r/raspbian-archive-keyring/raspbian-archive-keyring_20120528.2_all.deb && sudo dpkg -i raspbian-archive-keyring_20120528.2_all.deb - fi - if [ "${{ inputs.arch }}" = "aarch64" ]; then - multistrap_conf=multistrap_armbian64_buster.conf fi # Retry 2 times because Rasbian mirrors are often failing diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e00313679..ae42551f7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -637,12 +637,12 @@ jobs: needs: [ build-nodejs-Linux, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: # https://nodejs.org/en/about/releases/ nodejs-version: [12, 14, 16, 17] models: ["test"] samplerate: ["16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -689,11 +689,11 @@ jobs: needs: [ build-nodejs-Linux, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: electronjs-version: [12.0.0, 13.0.0, 14.0.0, 15.0.0, 16.0.0] models: ["test"] samplerate: ["16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -1169,6 +1169,18 @@ jobs: - uses: actions/download-artifact@v2 with: name: stt-tflite-3.7-aarch64.whl + - uses: actions/download-artifact@v2 + with: + name: stt-tflite-3.8-armv7.whl + - uses: actions/download-artifact@v2 + with: + name: stt-tflite-3.8-aarch64.whl + - uses: actions/download-artifact@v2 + with: + name: stt-tflite-3.9-armv7.whl + - uses: actions/download-artifact@v2 + with: + name: stt-tflite-3.9-aarch64.whl - name: Upload artifacts to GitHub release uses: ./.github/actions/upload-release-asset with: @@ -1914,12 +1926,12 @@ jobs: needs: [build-nodejs-macOS, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: # https://nodejs.org/en/about/releases/ nodejs-version: [12, 14, 16, 17] models: ["test"] samplerate: ["16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -2330,11 +2342,11 @@ jobs: needs: [ build-nodejs-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: nodejs-version: [12, 14, 16] models: ["test"] samplerate: ["16000"] - fail-fast: false env: CI_TMP_DIR: tmp/ STT_TEST_MODEL: tmp/output_graph.tflite @@ -2498,12 +2510,12 @@ jobs: needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: # https://nodejs.org/en/about/releases/ nodejs-version: [12, 17] models: ["test", "prod"] samplerate: ["8000", "16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -2550,11 +2562,11 @@ jobs: needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: electronjs-version: [12.0.0, 16.0.0] models: ["test", "prod"] samplerate: ["8000", "16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -2602,12 +2614,12 @@ jobs: needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: # https://nodejs.org/en/about/releases/ nodejs-version: [12, 17] models: ["test", "prod"] samplerate: ["8000", "16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -2652,11 +2664,11 @@ jobs: needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: electronjs-version: [12.0.0, 16.0.0] models: ["test", "prod"] samplerate: ["8000", "16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite @@ -2705,12 +2717,12 @@ jobs: needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: # https://nodejs.org/en/about/releases/ nodejs-version: [12, 16] models: ["test", "prod"] samplerate: ["8000", "16000"] - fail-fast: false env: CI_TMP_DIR: tmp/ STT_TEST_MODEL: tmp/output_graph.tflite @@ -2768,11 +2780,11 @@ jobs: needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: electronjs-version: [12.0.0, 16.0.0] models: ["test", "prod"] samplerate: ["8000", "16000"] - fail-fast: false env: CI_TMP_DIR: tmp/ STT_TEST_MODEL: tmp/output_graph.tflite @@ -2842,7 +2854,7 @@ jobs: - name: Install chroot uses: ./.github/actions/multistrap with: - arch: armv7 + multistrap-config: multistrap_raspbian_buster.conf - run: ./ci_scripts/tf-setup.sh - run: ./ci_scripts/armv7-build.sh - run: ./ci_scripts/package.sh @@ -2871,7 +2883,7 @@ jobs: - name: Install chroot uses: ./.github/actions/multistrap with: - arch: aarch64 + multistrap-config: multistrap_armbian64_buster.conf - run: ./ci_scripts/tf-setup.sh - run: ./ci_scripts/aarch64-build.sh - run: ./ci_scripts/package.sh @@ -2888,12 +2900,19 @@ jobs: runs-on: ubuntu-20.04 needs: [build-lib-LinuxArmv7, swig_Linux] strategy: + fail-fast: false matrix: - python-version: [3.7] + include: + - python-version: "3.7" + system-raspbian: multistrap-raspbian-buster + multistrap-config: multistrap_raspbian_buster.conf + - python-version: "3.9" + system-raspbian: multistrap-raspbian-bullseye + multistrap-config: multistrap_raspbian_bullseye.conf env: DEBIAN_FRONTEND: "noninteractive" SYSTEM_TARGET: rpi3 - SYSTEM_RASPBIAN: ${{ github.workspace }}/multistrap-raspbian-buster + SYSTEM_RASPBIAN: "${{ github.workspace }}/${{ matrix.system-raspbian }}" steps: - uses: actions/checkout@v2 with: @@ -2921,7 +2940,7 @@ jobs: - name: "Install chroot" uses: ./.github/actions/multistrap with: - arch: armv7 + multistrap-config: ${{ matrix.multistrap-config }} - id: get_numpy uses: ./.github/actions/numpy_vers with: @@ -2977,7 +2996,7 @@ jobs: - name: "Install chroot" uses: ./.github/actions/multistrap with: - arch: armv7 + multistrap-config: multistrap_raspbian_buster.conf - uses: actions/setup-node@v2 with: node-version: 12 @@ -3018,7 +3037,7 @@ jobs: needs: [build-lib-LinuxAarch64, swig_Linux] strategy: matrix: - python-version: [3.7] + python-version: ["3.7"] env: DEBIAN_FRONTEND: "noninteractive" SYSTEM_TARGET: rpi3-armv8 @@ -3052,7 +3071,7 @@ jobs: - name: "Install chroot" uses: ./.github/actions/multistrap with: - arch: aarch64 + multistrap-config: multistrap_armbian64_buster.conf - id: get_numpy uses: ./.github/actions/numpy_vers with: @@ -3108,7 +3127,7 @@ jobs: - name: "Install chroot" uses: ./.github/actions/multistrap with: - arch: aarch64 + multistrap-config: multistrap_armbian64_buster.conf - uses: actions/setup-node@v2 with: node-version: 12 @@ -3148,12 +3167,22 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: - arch: [ "armv7", "aarch64" ] + include: + - arch: "armv7" + system: "buster" + multistrap-config: "multistrap_raspbian_buster.conf" + - arch: "armv7" + system: "bullseye" + multistrap-config: "multistrap_raspbian_bullseye.conf" + - arch: "aarch64" + system: "buster" + multistrap-config: "multistrap_armbian64_buster.conf" env: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" - SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }} + SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}-${{ matrix.system }} steps: - uses: actions/checkout@v2 with: @@ -3161,14 +3190,14 @@ jobs: - name: "Install and setup chroot" uses: ./.github/actions/multistrap with: - arch: ${{ matrix.arch }} + multistrap-config: ${{ matrix.multistrap-config }} packages: "bash wget curl sox xxd libatlas3-base libopenblas-base ca-certificates python3 python3-pip gnupg libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libgbm1 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcb-dri3-0 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xauth" - name: "Create a chroot tarball" run: | sudo tar -cf - -C ${{ env.SYSTEM_RASPBIAN }}/ --one-file-system . | xz -9 -T0 > ${{ github.workspace }}/chroot.tar.xz - uses: actions/upload-artifact@v2 with: - name: chroot-${{ matrix.arch }} + name: chroot-${{ matrix.arch }}-${{ matrix.system }} path: ${{ github.workspace }}/chroot.tar.xz test-cpp-LinuxArm: name: "LinArm*|Test C++ binary" @@ -3176,15 +3205,62 @@ jobs: needs: [ build-lib-LinuxArmv7, build-lib-LinuxAarch64, train-test-model-Linux, build-test-chroot ] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: - arch: [ "armv7", "aarch64" ] - models: ["test", "prod"] - samplerate: ["8000", "16000"] + include: + - arch: "armv7" + system: "buster" + models: "test" + samplerate: "8000" + - arch: "armv7" + system: "buster" + models: "test" + samplerate: "16000" + - arch: "armv7" + system: "buster" + models: "prod" + samplerate: "8000" + - arch: "armv7" + system: "buster" + models: "prod" + samplerate: "16000" + - arch: "armv7" + system: "bullseye" + models: "test" + samplerate: "8000" + - arch: "armv7" + system: "bullseye" + models: "test" + samplerate: "16000" + - arch: "armv7" + system: "bullseye" + models: "prod" + samplerate: "8000" + - arch: "armv7" + system: "bullseye" + models: "prod" + samplerate: "16000" + - arch: "aarch64" + system: "buster" + models: "test" + samplerate: "8000" + - arch: "aarch64" + system: "buster" + models: "test" + samplerate: "16000" + - arch: "aarch64" + system: "buster" + models: "prod" + samplerate: "8000" + - arch: "aarch64" + system: "buster" + models: "prod" + samplerate: "16000" env: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite - SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }} + SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}-${{ matrix.system }} steps: - name: "Install QEMU" run: | @@ -3195,7 +3271,7 @@ jobs: fetch-depth: 1 - uses: actions/download-artifact@v2 with: - name: "chroot-${{ matrix.arch }}" + name: "chroot-${{ matrix.arch }}-${{ matrix.system }}" path: ${{ env.CI_TMP_DIR }}/ - run: | mkdir ${{ env.SYSTEM_RASPBIAN }}/ @@ -3234,16 +3310,74 @@ jobs: needs: [ build-python-LinuxArmv7, build-python-LinuxAarch64, train-test-model-Linux, build-test-chroot ] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: - arch: [ "armv7", "aarch64" ] - python-version: [3.7] - models: ["test", "prod"] - samplerate: ["8000", "16000"] + include: + - arch: "armv7" + system: "buster" + python-version: "3.7" + models: "test" + samplerate: "8000" + - arch: "armv7" + system: "buster" + python-version: "3.7" + models: "test" + samplerate: "16000" + - arch: "armv7" + system: "buster" + python-version: "3.7" + models: "prod" + samplerate: "8000" + - arch: "armv7" + system: "buster" + python-version: "3.7" + models: "prod" + samplerate: "16000" + - arch: "armv7" + system: "bullseye" + python-version: "3.9" + models: "test" + samplerate: "8000" + - arch: "armv7" + system: "bullseye" + python-version: "3.9" + models: "test" + samplerate: "16000" + - arch: "armv7" + system: "bullseye" + python-version: "3.9" + models: "prod" + samplerate: "8000" + - arch: "armv7" + system: "bullseye" + python-version: "3.9" + models: "prod" + samplerate: "16000" + - arch: "aarch64" + system: "buster" + python-version: "3.7" + models: "test" + samplerate: "8000" + - arch: "aarch64" + system: "buster" + python-version: "3.7" + models: "test" + samplerate: "16000" + - arch: "aarch64" + system: "buster" + python-version: "3.7" + models: "prod" + samplerate: "8000" + - arch: "aarch64" + system: "buster" + python-version: "3.7" + models: "prod" + samplerate: "16000" env: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite - SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }} + SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}-${{ matrix.system }} PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple https://lissyx.github.io/deepspeech-python-wheels/" steps: - name: "Install QEMU" @@ -3255,7 +3389,7 @@ jobs: fetch-depth: 1 - uses: actions/download-artifact@v2 with: - name: "chroot-${{ matrix.arch }}" + name: "chroot-${{ matrix.arch }}-${{ matrix.system }}" path: ${{ env.CI_TMP_DIR }}/ - run: | mkdir ${{ env.SYSTEM_RASPBIAN }}/ @@ -3293,18 +3427,18 @@ jobs: needs: [ build-nodejs-LinuxArmv7, build-nodejs-LinuxAarch64, train-test-model-Linux, build-test-chroot ] if: ${{ github.event_name == 'pull_request' }} strategy: + fail-fast: false matrix: - arch: [ "armv7", "aarch64" ] + arch: ["armv7", "aarch64"] # https://nodejs.org/en/about/releases/ nodejs-version: [12, 14, 16, 17] models: ["test"] samplerate: ["16000"] - fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite - SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }} + SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}-buster steps: - name: "Install QEMU" run: | @@ -3315,7 +3449,7 @@ jobs: fetch-depth: 1 - uses: actions/download-artifact@v2 with: - name: "chroot-${{ matrix.arch }}" + name: "chroot-${{ matrix.arch }}-buster" path: ${{ env.CI_TMP_DIR }}/ - run: | mkdir ${{ env.SYSTEM_RASPBIAN }}/ @@ -3356,8 +3490,9 @@ jobs: # Disable this task because it seems qemu does not work super-well with ElectronJS if: ${{ github.event_name == 'disabled' }} strategy: + fail-fast: false matrix: - arch: [ "armv7", "aarch64" ] + arch: ["armv7", "aarch64"] electronjs-version: [12.0.0, 13.0.0, 14.0.0, 15.0.0, 16.0.0] models: ["test"] samplerate: ["16000"] @@ -3365,7 +3500,7 @@ jobs: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.tflite - SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }} + SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}-buster DISPLAY: ":99.0" steps: - name: "Install QEMU" @@ -3377,7 +3512,7 @@ jobs: fetch-depth: 1 - uses: actions/download-artifact@v2 with: - name: "chroot-${{ matrix.arch }}" + name: "chroot-${{ matrix.arch }}-buster" path: ${{ env.CI_TMP_DIR }}/ - run: | mkdir ${{ env.SYSTEM_RASPBIAN }}/ diff --git a/.gitignore b/.gitignore index 3bc605f85..baa3a1309 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea/ .ipynb_checkpoints *.pyc *.swp diff --git a/Dockerfile.build b/Dockerfile.build index b97034c84..84fa5954e 100644 --- a/Dockerfile.build +++ b/Dockerfile.build @@ -6,7 +6,7 @@ FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 # >> START Install base software # Get basic packages -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends --fix-broken \ apt-utils \ bash-completion \ build-essential \ @@ -159,6 +159,7 @@ RUN make NUM_PROCESSES=$(nproc) stt WORKDIR /STT RUN cd native_client/python && make NUM_PROCESSES=$(nproc) bindings +RUN pip3 install --upgrade pip RUN pip3 install -U pip setuptools wheel RUN pip3 install --upgrade native_client/python/dist/*.whl diff --git a/native_client/BUILD b/native_client/BUILD index 4127108ca..b0a8d890b 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -8,6 +8,20 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_c_module", "swift_libra load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary") +config_setting( + name = "rpi3", + define_values = { + "target_system": "rpi3" + }, +) + +config_setting( + name = "rpi3-armv8", + define_values = { + "target_system": "rpi3-armv8" + }, +) + # Only required until https://github.com/emscripten-core/emsdk/pull/1060 # gets merged. Once that's done, we could use "@platform//cpu:wasm32". config_setting( @@ -15,7 +29,6 @@ config_setting( values = {"cpu": "wasm"}, ) - genrule( name = "workspace_status", outs = ["workspace_status.cc"], diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 26ff96077..df5b4ab7b 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -78,7 +78,7 @@ PYVER := $(shell python -c "import platform; maj, min, _ = platform.python_versi PYTHON_PACKAGES := PYTHON_PATH := PYTHONPATH=$(RASPBIAN)/usr/lib/python$(PYVER)/:$(RASPBIAN)/usr/lib/python3/dist-packages/ NUMPY_INCLUDE := NUMPY_INCLUDE=$(RASPBIAN)/usr/include/python3.7m/ -PYTHON_SYSCONFIGDATA := _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata_m_linux_arm-linux-gnueabihf +PYTHON_SYSCONFIGDATA := _PYTHON_SYSCONFIGDATA_NAME=_sysconfigdata__x86_64-linux-gnu PYTHON_PLATFORM_NAME := --plat-name linux_armv7l NODE_PLATFORM_TARGET := --target_arch=arm --target_platform=linux TOOLCHAIN_LDD_OPTS := --root $(RASPBIAN)/ diff --git a/native_client/multistrap_raspbian_bullseye.conf b/native_client/multistrap_raspbian_bullseye.conf new file mode 100644 index 000000000..78211dffe --- /dev/null +++ b/native_client/multistrap_raspbian_bullseye.conf @@ -0,0 +1,14 @@ +[General] +arch=armhf +noauth=false +unpack=true +debootstrap=Raspbian +aptsources=Raspbian +cleanup=true + +[Raspbian] +packages=apt libc6 libc6-dev libffi-dev libstdc++-9-dev linux-libc-dev libpython3.9-dev python3.9-distutils libsox-dev python3-numpy +source=http://raspbian.raspberrypi.org/raspbian/ +keyring=raspbian-archive-keyring +components=main +suite=bullseye