python-venv-package.yml

name: Build and Package Python Virtual Environment

on:
  workflow_dispatch:
    inputs:
      model_dir:
        description: "Path to model directory in github repo"
        required: true
      model_name:
        description: "name of model to be release"
        required: true
      repo_name:
        description: "name of repo to be checked out"
        required: true
      branch_name:
        description: "name of branch to be checked out"
        required: true
        default: main
      hf_repo:
        description: "name of huggingface repo to be pushed"
        required: true
      hf_prefix_branch: 
        description: "prefix of hf branch"
        required: false



env:
  MODEL_DIR: ${{ inputs.model_dir }}
  MODEL_NAME:  ${{ inputs.model_name }}
  REPO_NAME:  ${{ inputs.repo_name }}
  BRANCH_NAME:  ${{ inputs.branch_name }}
  HF_REPO: ${{ inputs.hf_repo }}
  HF_PREFIX_BRANCH: ${{ inputs.hf_prefix_branch }}

jobs:
  build-and-test:
    runs-on: ${{ matrix.runs-on }}
    timeout-minutes: 3600
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: "linux"
            name: "amd64"
            runs-on: "ubuntu-20-04-cuda-12-0"
          - os: "mac"
            name: "amd64"
            runs-on: "macos-selfhosted-12"
          - os: "mac"
            name: "arm64"
            runs-on: "macos-selfhosted-12-arm64"
          - os: "windows"
            name: "amd64"
            runs-on: "windows-cuda-12-0"
    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v3
        with:
          submodules: recursive
          repository: ${{env.REPO_NAME}}
          ref: ${{env.BRANCH_NAME}}
      - uses: conda-incubator/setup-miniconda@v3
        if: runner.os != 'windows'
        with:
          auto-update-conda: true
          python-version: 3.11
      - name: use python
        if : runner.os == 'windows'
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Get Cer for code signing
        if: runner.os == 'macOS'
        run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12
        shell: bash
        env:
          CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
  
      - uses: apple-actions/import-codesign-certs@v2
        continue-on-error: true
        if: runner.os == 'macOS'
        with:
          p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }}
          p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }}

      - name: Get Cer for code signing
        if: runner.os == 'macOS'
        run: base64 -d <<< "$NOTARIZE_P8_BASE64" > /tmp/notary-key.p8
        shell: bash
        env:
          NOTARIZE_P8_BASE64: ${{ secrets.NOTARIZE_P8_BASE64 }}

      - name: Install dependencies Windows
        if: runner.os == 'windows'
        shell: pwsh
        run: |
          python3 -m pip install fastapi
          python3 -m pip freeze | % { python3 -m pip uninstall -y $_ }
          python3 -m pip install --upgrade pip
          python3 -m pip install -I -r ${{env.MODEL_DIR}}/requirements.cuda.txt
          python3 -m pip install python-dotenv
      - name: Install dependencies Linux
        if: runner.os == 'linux'
        run: |
          conda create -y -n ${{env.MODEL_NAME}} python=3.11
          source $HOME/miniconda3/bin/activate base
          conda init
          conda activate ${{env.MODEL_NAME}}
          python -m pip install fastapi
          python -m pip freeze | xargs python -m pip uninstall -y
          python -m pip install --upgrade pip
          python -m pip install -r ${{env.MODEL_DIR}}/requirements.cuda.txt
          python -m pip install python-dotenv
      - name: Install dependencies Mac
        if: runner.os == 'macOS'
        run: |
          conda create -y -n ${{env.MODEL_NAME}} python=3.11
          source $HOME/miniconda3/bin/activate base
          conda init
          conda activate ${{env.MODEL_NAME}}
          python -m pip install fastapi
          python -m pip freeze | xargs python -m pip uninstall -y
          python -m pip install --upgrade pip
          python -m pip install -r ${{env.MODEL_DIR}}/requirements.txt
          python -m pip install python-dotenv

      - name: prepare python package windows
        if : runner.os == 'windows'
        shell: pwsh
        run: |
            $pythonPath = where.exe python
            echo "Python path (where.exe): $pythonPath"
            $pythonFolder = Split-Path -Path "$pythonPath" -Parent
            echo "PYTHON_FOLDER=$pythonFolder" >> $env:GITHUB_ENV
            copy "$pythonFolder\python*.*" "$pythonFolder\Scripts\"

      - name: prepare python package macos
        if : runner.os == 'macOs'
        run: |
            source $HOME/miniconda3/bin/activate base
            conda init
            conda activate ${{env.MODEL_NAME}}
            PYTHON_PATH=$(which python)
            echo $PYTHON_PATH
            PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH"))
            echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV
            echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}"
      - name: prepare python package linux
        if : runner.os == 'linux'
        run: |
            source $HOME/miniconda3/bin/activate base
            conda init
            conda activate ${{env.MODEL_NAME}}
            PYTHON_PATH=$(which python)
            echo $PYTHON_PATH
            PYTHON_FOLDER=$(dirname $(dirname "$PYTHON_PATH"))
            rm -rf $PYTHON_FOLDER/lib/python3.1
            echo "PYTHON_FOLDER=$PYTHON_FOLDER" >> $GITHUB_ENV
            echo "github end PYTHON_FOLDER: ${{env.PYTHON_FOLDER}}"

      - name: create plist file
        if: runner.os == 'macOS'
        run: |
          cat << EOF > /tmp/entitlements.plist
          <?xml version="1.0" encoding="UTF-8"?>
          <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
          <plist version="1.0">
          <dict>
              <!-- These are required for binaries built by PyInstaller -->
              <key>com.apple.security.cs.allow-jit</key>
              <true/>
              <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
              <true/>

              <!-- Add these for additional permissions -->
              <key>com.apple.security.app-sandbox</key>
              <false/>
              <key>com.apple.security.network.client</key>
              <true/>
              <key>com.apple.security.network.server</key>
              <true/>
              <key>com.apple.security.device.audio-input</key>
              <true/>
              <key>com.apple.security.device.microphone</key>
              <true/>
              <key>com.apple.security.device.camera</key>
              <true/>
              <key>com.apple.security.files.user-selected.read-write</key>
              <true/>
              <key>com.apple.security.cs.disable-library-validation</key>
              <true/>
              <key>com.apple.security.cs.allow-dyld-environment-variables</key>
              <true/>
              <key>com.apple.security.cs.allow-executable-memory</key>
              <true/>
          </dict>
          </plist>
          EOF

      - name: Notary macOS Binary
        if: runner.os == 'macOS'
        run: |
          codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python
          codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime ${{env.PYTHON_FOLDER}}/bin/python3
          # Code sign all .so files and .dylib files
          
          find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec codesign --force --entitlements="/tmp/entitlements.plist" -s "${{ secrets.DEVELOPER_ID }}" --options=runtime {} \;

          curl -sSfL https://raw.githubusercontent.com/anchore/quill/main/install.sh | sudo sh -s -- -b /usr/local/bin
          # Notarize the binary
          quill notarize ${{env.PYTHON_FOLDER}}/bin/python
          quill notarize ${{env.PYTHON_FOLDER}}/bin/python3
          find ${{env.PYTHON_FOLDER}} -type f \( -name "*.so" -o -name "*.dylib" \) -exec quill notarize {} \;
        env:
          QUILL_NOTARY_KEY_ID: ${{ secrets.NOTARY_KEY_ID }}
          QUILL_NOTARY_ISSUER: ${{ secrets.NOTARY_ISSUER }}
          QUILL_NOTARY_KEY: "/tmp/notary-key.p8"


      - name: Upload Artifact MacOS
        if : runner.os == 'macOS'
        run: |
          brew install zip
          cd ${{env.PYTHON_FOLDER}} && zip -r venv.zip *
          conda create -y -n hf-upload python=3.11
          source $HOME/miniconda3/bin/activate base
          conda init
          conda activate hf-upload
          python -m pip install hf-transfer huggingface_hub
          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential
          huggingface-cli upload ${{env.HF_REPO}} venv.zip --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }}
          rm -rf venv.zip
          huggingface-cli logout

      - name: Upload Artifact Linux
        if : runner.os == 'linux'
        run: |
          sudo apt-get install -y zip
          cd ${{env.PYTHON_FOLDER}} && zip -r venv.zip *
          conda create -y -n hf-upload python=3.11
          source $HOME/miniconda3/bin/activate base
          conda init
          conda activate hf-upload
          python -m pip install hf-transfer huggingface_hub
          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential
          huggingface-cli upload ${{env.HF_REPO}} venv.zip --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }}
          rm -rf venv.zip
          huggingface-cli logout


      - name: Upload Artifact Windows
        if : runner.os == 'windows'
        shell: pwsh
        run: |
          Compress-Archive -Path ${{env.PYTHON_FOLDER}}/* -DestinationPath venv.zip
          python -m pip install hf-transfer huggingface_hub
          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential
          huggingface-cli upload ${{env.HF_REPO}} venv.zip --revision ${{env.HF_PREFIX_BRANCH}}-${{ matrix.os }}-${{ matrix.name }}
          rm venv.zip
          huggingface-cli logout


      - name: Post Upload windows
        if : runner.os == 'windows'
        run: |
          rm ${{env.PYTHON_FOLDER}}/Scripts/python*.*

      - name: Remove Keychain
        continue-on-error: true
        if: always() && runner.os == 'macOS'
        run: |
          security delete-keychain signing_temp.keychain