feat: package env for whispervq #86
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Package Python Library | |
| on: | |
| push: | |
| branches: [ feat/python-package-ci ] | |
| workflow_dispatch: | |
| inputs: | |
| model_dir: | |
| description: "Path to model directory in janhq/models repo" | |
| required: true | |
| model_name: | |
| description: "name of model to be release" | |
| required: true | |
| repo_name: | |
| description: "name of repo to be checked out" | |
| required: true | |
| branch_name: | |
| description: "name of branch to be checked out" | |
| required: true | |
| default: main | |
| env: | |
| MODEL_DIR: models/whispervq # ${{ inputs.model_dir }} | |
| MODEL_NAME: whispervq # ${{ inputs.model_name }} | |
| REPO_NAME: janhq/models # ${{ inputs.model_name }} | |
| BRANCH_NAME: feat/ci-python-models # ${{ inputs.model_name }} | |
| jobs: | |
| build-and-test: | |
| runs-on: ${{ matrix.runs-on }} | |
| timeout-minutes: 360 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: "linux" | |
| name: "amd64" | |
| runs-on: "ubuntu-20-04-cuda-12-0" | |
| # - os: "mac" | |
| # name: "amd64" | |
| # runs-on: "macos-selfhosted-12" | |
| # - os: "mac" | |
| # name: "arm64" | |
| # runs-on: "macos-selfhosted-12-arm64" | |
| - os: "windows" | |
| name: "amd64" | |
| runs-on: "windows-cuda-12-0" | |
| steps: | |
| - name: Clone | |
| id: checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| repository: ${{env.REPO_NAME}} | |
| ref: ${{env.BRANCH_NAME}} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| if: runner.os != 'windows' | |
| with: | |
| auto-update-conda: true | |
| python-version: 3.11 | |
| - name: use python | |
| if : runner.os == 'windows' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Get Cer for code signing | |
| if: runner.os == 'macOS' | |
| run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12 | |
| shell: bash | |
| env: | |
| CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }} | |
| - uses: apple-actions/import-codesign-certs@v2 | |
| continue-on-error: true | |
| if: runner.os == 'macOS' | |
| with: | |
| p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} | |
| p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} | |
| - name: Get Cer for code signing | |
| if: runner.os == 'macOS' | |
| run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8 | |
| shell: bash | |
| env: | |
| NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }} | |
| - name: Install dependencies Windows | |
| if: runner.os == 'windows' | |
| shell: pwsh | |
| run: | | |
| python3 -m pip install fastapi | |
| python3 -m pip freeze | % { python3 -m pip uninstall -y $_ } | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install -I -r ${{env.MODEL_DIR}}/requirements.cuda.txt | |
| python3 -m pip install python-dotenv | |
| - name: Install dependencies Linux | |
| if: runner.os == 'linux' | |
| run: | | |
| conda create -y -n ${{env.MODEL_NAME}} python=3.11 | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| python -m pip install fastapi | |
| python -m pip freeze | xargs python -m pip uninstall -y | |
| python -m pip install --upgrade pip | |
| python -m pip install -r ${{env.MODEL_DIR}}/requirements.cuda.txt | |
| python -m pip install python-dotenv | |
| - name: Install dependencies Mac | |
| if: runner.os == 'macOS' | |
| run: | | |
| conda create -y -n ${{env.MODEL_NAME}} python=3.11 | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| python -m pip install fastapi | |
| python -m pip freeze | xargs python -m pip uninstall -y | |
| python -m pip install --upgrade pip | |
| python -m pip install -r ${{env.MODEL_DIR}}/requirements.txt | |
| python -m pip install python-dotenv | |
| - name: prepare python package windows | |
| if : runner.os == 'windows' | |
| shell: pwsh | |
| run: | | |
| $pythonPath = where.exe python | |
| echo "Python path (where.exe): $pythonPath" | |
| $pythonFolder = Split-Path -Path "$pythonPath" -Parent | |
| echo "PYTHON_FOLDER=$pythonFolder" >> $env:GITHUB_ENV | |
| copy "$pythonFolder\python*.*" "$pythonFolder\Scripts\" | |
| - name: prepare python package macos | |
| if : runner.os == 'macOs' | |
| run: | | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| PYTHON_PATH=$(which python) | |
| echo $PYTHON_PATH | |
| PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH")) | |
| echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV | |
| echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" | |
| - name: prepare python package linux | |
| if : runner.os == 'linux' | |
| run: | | |
| source $HOME/miniconda3/bin/activate base | |
| conda init | |
| conda activate ${{env.MODEL_NAME}} | |
| PYTHON_PATH=$(which python) | |
| echo $PYTHON_PATH | |
| PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH")) | |
| rm -rf $PYTHON_FOLDER/lib/python3.1 | |
| echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV | |
| echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}" | |
| - name: create plist file | |
| if: runner.os == 'macOS' | |
| run: | | |
| cat << EOF > /tmp/entitlements.plist | |
| <?xml version="1.0" encoding="UTF-8"?> | |
| <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> | |
| <plist version="1.0"> | |
| <dict> | |
| <!-- These are required for binaries built by PyInstaller --> | |
| <key>com.apple.security.cs.allow-jit</key> | |
| <true/> | |
| <key>com.apple.security.cs.allow-unsigned-executable-memory</key> | |
| <true/> | |
| <!-- Add these for additional permissions --> | |
| <key>com.apple.security.app-sandbox</key> | |
| <false/> | |
| <key>com.apple.security.network.client</key> | |
| <true/> | |
| <key>com.apple.security.network.server</key> | |
| <true/> | |
| <key>com.apple.security.device.audio-input</key> | |
| <true/> | |
| <key>com.apple.security.device.microphone</key> | |
| <true/> | |
| <key>com.apple.security.device.camera</key> | |
| <true/> | |
| <key>com.apple.security.files.user-selected.read-write</key> | |
| <true/> | |
| <key>com.apple.security.cs.disable-library-validation</key> | |
| <true/> | |
| <key>com.apple.security.cs.allow-dyld-environment-variables</key> | |
| <true/> | |
| <key>com.apple.security.cs.allow-executable-memory</key> | |
| <true/> | |
| </dict> | |
| </plist> | |
| EOF | |
| - name: Notary macOS Binary | |
| if: runner.os == 'macOS' | |
| run: | | |
| codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python | |
| codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3 | |
| # Code sign all .so files and .dylib files | |
| find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \; | |
| curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sudo sh -s -- -b /usr/local/bin | |
| # Notarize the binary | |
| quill notarize ${{env.PYTHON_FOLDER}}/bin/python | |
| quill notarize ${{env.PYTHON_FOLDER}}/bin/python3 | |
| find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec quill notarize {} \; | |
| env: | |
| QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }} | |
| QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }} | |
| QUILL_NOTARY_KEY: "/tmp/notary-key.p8" | |
| - name: Upload Artifact | |
| #if : runner.os == 'windows' || runner.os == 'linux' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{env.MODEL_NAME}}-${{ matrix.os }}-${{ matrix.name }} | |
| path: ${{env.PYTHON_FOLDER}} | |
| include-hidden-files: true | |
| compression-level: 9 | |
| - name: Post Upload windows | |
| if : runner.os == 'windows' | |
| run: | | |
| rm ${{env.PYTHON_FOLDER}}/Scripts/python*.* | |
| - name: Remove Keychain | |
| continue-on-error: true | |
| if: always() && runner.os == 'macOS' | |
| run: | | |
| security delete-keychain signing_temp.keychain |