Fix workflows nightly-wheels and coverity (#1531)

Add a GitHub composite action that can be used in other workflows to build and install Triton leveraging Triton and LLVM cahces. Use the composite action in workflows nightly-triton and coverity. Continues #1510.
intel · Jun 29, 2024 · a006a5c · a006a5c
1 parent fd06146
commit a006a5c
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 90 deletions.
diff --git a/.github/actions/setup-triton/action.yml b/.github/actions/setup-triton/action.yml
@@ -0,0 +1,91 @@
+# Builds and installs Triton. Uses git clone in the current directory.
+# Sets the following environment variables:
+# * LLVM_COMMIT_ID
+description: Build and install IPEX wheels
+inputs:
+  build_llvm:
+    description: Build LLVM
+    default: false
+  command:
+    description: Command to execute
+    default: DEBUG=1 pip install --no-build-isolation '.[build,tests,tutorials]'
+runs:
+  using: "composite"
+  steps:
+    - name: Get LLVM commit id
+      shell: bash
+      run: |
+        LLVM_COMMIT_ID=$(<cmake/llvm-hash.txt)
+        echo "LLVM_COMMIT_ID=$LLVM_COMMIT_ID" | tee -a $GITHUB_ENV
+
+    - name: Load LLVM cache
+      if: ${{ inputs.build_llvm == 'true' }}
+      id: llvm-cache
+      uses: ./.github/actions/load
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 2
+      with:
+        path: $HOME/packages
+        key: packages-${{ env.LLVM_COMMIT_ID }}-${{ env.CACHE_NUMBER }}
+
+    - name: Clone LLVM
+      if: ${{ inputs.build_llvm == 'true' && steps.llvm-cache.outputs.status == 'miss' }}
+      uses: actions/checkout@v4
+      with:
+        repository: llvm/llvm-project
+        ref: ${{ env.LLVM_COMMIT_ID }}
+        path: llvm
+        submodules: recursive
+
+    - name: Build LLVM
+      if: ${{ inputs.build_llvm == 'true' && steps.llvm-cache.outputs.status == 'miss' }}
+      shell: bash
+      run: |
+        export BASE=$HOME
+        ln -s $PWD/llvm $BASE/llvm
+        ./scripts/compile-triton.sh --llvm
+
+    - name: Set LLVM_SYSPATH
+      if: ${{ inputs.build_llvm == 'true' }}
+      shell: bash
+      run: |
+        echo "LLVM_SYSPATH=$HOME/packages/llvm" | tee -a $GITHUB_ENV
+
+    - name: Save LLVM cache
+      if: ${{ inputs.build_llvm == 'true' && steps.llvm-cache.outputs.status == 'miss' }}
+      uses: ./.github/actions/save
+      with:
+        path: ${{ steps.llvm-cache.outputs.path }}
+        dest: ${{ steps.llvm-cache.outputs.dest }}
+
+    - name: Prepare Triton cache
+      shell: bash
+      run: |
+        mkdir -p ~/.triton
+        TRITON_CACHE_KEY="$(sha256sum python/setup.py | cut -d\  -f1)"
+        echo "TRITON_CACHE_KEY=$TRITON_CACHE_KEY" | tee -a $GITHUB_ENV
+
+    - name: Load Triton cache
+      id: triton-cache
+      uses: ./.github/actions/load
+      env:
+        # Increase this value to reset cache
+        CACHE_NUMBER: 1
+      with:
+        path: $HOME/.triton/nvidia
+        key: triton-nvidia-${{ env.TRITON_CACHE_KEY }}-${{ env.CACHE_NUMBER }}
+
+    - name: Build Triton
+      shell: bash
+      run: |
+        cd python
+        pip install wheel
+        ${{ inputs.command }}
+
+    - name: Save Triton cache
+      if: steps.triton-cache.outputs.status == 'miss'
+      uses: ./.github/actions/save
+      with:
+        path: ${{ steps.triton-cache.outputs.path }}
+        dest: ${{ steps.triton-cache.outputs.dest }}
diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml
@@ -95,50 +95,6 @@ jobs:
         with:
           python-version: ${{ inputs.python_version }}
 
-      - name: Get LLVM commit id
-        run: |
-          LLVM_COMMIT_ID=$(<cmake/llvm-hash.txt)
-          echo "LLVM_COMMIT_ID=$LLVM_COMMIT_ID" | tee -a $GITHUB_ENV
-
-      - name: Load LLVM cache
-        if: inputs.build_llvm
-        id: llvm-cache
-        uses: ./.github/actions/load
-        env:
-          # Increase this value to reset cache
-          CACHE_NUMBER: 2
-        with:
-          path: $HOME/packages
-          key: packages-${{ env.LLVM_COMMIT_ID }}-${{ env.CACHE_NUMBER }}
-
-      - name: Clone LLVM
-        if: inputs.build_llvm && steps.llvm-cache.outputs.status == 'miss'
-        uses: actions/checkout@v4
-        with:
-          repository: llvm/llvm-project
-          ref: ${{ env.LLVM_COMMIT_ID }}
-          path: llvm
-          submodules: recursive
-
-      - name: Build LLVM
-        if: inputs.build_llvm && steps.llvm-cache.outputs.status == 'miss'
-        run: |
-          export BASE=$HOME
-          ln -s $PWD/llvm $BASE/llvm
-          ./scripts/compile-triton.sh --llvm
-
-      - name: Set LLVM_SYSPATH
-        if: inputs.build_llvm
-        run: |
-          echo "LLVM_SYSPATH=$HOME/packages/llvm" | tee -a $GITHUB_ENV
-
-      - name: Save LLVM cache
-        if: inputs.build_llvm && steps.llvm-cache.outputs.status == 'miss'
-        uses: ./.github/actions/save
-        with:
-          path: ${{ steps.llvm-cache.outputs.path }}
-          dest: ${{ steps.llvm-cache.outputs.dest }}
-
       - name: Setup PyTorch with IPEX
         if: ${{ inputs.install_ipex }}
         uses: ./.github/actions/setup-pytorch
@@ -162,36 +118,15 @@ jobs:
         if: ${{ !inputs.install_ipex }}
         uses: ./.github/actions/setup-fake-ipex
 
-      - name: Prepare Triton cache
+      - name: Install test dependencies
         run: |
-          mkdir -p ~/.triton
-          TRITON_CACHE_KEY="$(sha256sum python/setup.py | cut -d\  -f1)"
-          echo "TRITON_CACHE_KEY=$TRITON_CACHE_KEY" | tee -a $GITHUB_ENV
-
-      - name: Load Triton cache
-        id: triton-cache
-        uses: ./.github/actions/load
-        env:
-          # Increase this value to reset cache
-          CACHE_NUMBER: 1
-        with:
-          path: $HOME/.triton/nvidia
-          key: triton-nvidia-${{ env.TRITON_CACHE_KEY }}-${{ env.CACHE_NUMBER }}
-
-      - name: Build Triton
-        run: |
-          export DEBUG=1
-          cd python
-          pip install wheel pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout
-          pip install --no-build-isolation '.[build,tests,tutorials]'
+          pip install pytest pytest-xdist pytest-rerunfailures pytest-select pytest-timeout
           pip install git+https://github.com/kwasd/[email protected]
 
-      - name: Save Triton cache
-        if: steps.triton-cache.outputs.status == 'miss'
-        uses: ./.github/actions/save
+      - name: Setup Triton
+        uses: ./.github/actions/setup-triton
         with:
-          path: ${{ steps.triton-cache.outputs.path }}
-          dest: ${{ steps.triton-cache.outputs.dest }}
+          build_llvm: ${{ inputs.build_llvm }}
 
       - name: Run lit tests
         run: |

diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
@@ -35,7 +35,8 @@ jobs:
           CACHE_NUMBER: 1
         with:
           path: $HOME/coverity
-          key: coverity-$CACHE_NUMBER
+          # Update coverity each month
+          key: coverity-$(date +%Y%m)-$CACHE_NUMBER
 
       - name: Download coverity
         if: ${{ steps.coverity-cache.outputs.status == 'miss' }}
@@ -63,15 +64,14 @@ jobs:
           dest: ${{ steps.coverity-cache.outputs.dest }}
 
       - name: Run coverity build
-        run: |
-          pip install wheel
-          cd python
-          cov-build --dir $HOME/cov-int pip install -e .
-          tail $HOME/cov-int/build-log.txt
+        uses: ./.github/actions/setup-triton
+        with:
+          command: cov-build --dir $HOME/cov-int pip install -e .
 
       - name: Create coverity results tarball
         run: |
           cd $HOME
+          tail cov-int/build-log.txt
           tar zcf cov-int.tgz cov-int
 
       - name: Version for coverity build

diff --git a/.github/workflows/nightly-wheels.yml b/.github/workflows/nightly-wheels.yml
@@ -52,11 +52,6 @@ jobs:
       - name: Setup IPEX
         uses: ./.github/actions/setup-ipex
 
-      - name: Get LLVM commit id
-        run: |
-          LLVM_COMMIT_ID=$(<cmake/llvm-hash.txt)
-          echo "LLVM_COMMIT_ID=$LLVM_COMMIT_ID" >> $GITHUB_ENV
-
       - name: Identify Triton commit id
         run: |
           echo "TRITON_COMMIT_ID=$(git rev-parse HEAD)" >> $GITHUB_ENV
@@ -76,15 +71,12 @@ jobs:
 
       - name: Build Triton wheels
         if: ${{ steps.triton-cache.outputs.status == 'miss' }}
-        run: |
-          export DEBUG=1
-          export TRITON_WHEEL_VERSION_SUFFIX="+git$(git rev-parse --short HEAD)"
-          cd python
-          python setup.py bdist_wheel
-
-      - name: Install Triton
-        run: |
-          pip install python/dist/*.whl
+        uses: ./.github/actions/setup-triton
+        with:
+          command: >
+            DEBUG=1
+            TRITON_WHEEL_VERSION_SUFFIX="+git$(git rev-parse --short HEAD)"
+            python setup.py bdist_wheel && pip install dist/*.whl
 
       - name: Save Triton wheels to a cache
         if: ${{ steps.triton-cache.outputs.status == 'miss' }}