diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 054a7792e..1070025f4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -501,25 +501,30 @@ jobs:
       - name: Check out repo
         uses: actions/checkout@v4
       - name: Check python matches black format
+        id: black_check
         continue-on-error: true
         run: |
           cd python/interpret-core
           python -m pip install --upgrade black
           black --check .
+        shell: bash
+      - name: Capture exit code
+        id: capture_exit_code
+        run: echo "::set-output name=exitcode::${{ steps.black_check.outcome == 'failure' }}"
       - name: Mark step with a warning
-        #if: ${{ steps.black_check.outcome == 'failure' }}
+        if: steps.capture_exit_code.outputs.exitcode == 'true'
         uses: actions/github-script@v6
         with:
           script: |
             github.rest.checks.create({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              name: 'Warning: Step might have failed',
+              name: 'Warning: Failed black formatting check',
               head_sha: context.sha,
               status: 'completed',
               conclusion: 'neutral',
               output: {
                 title: 'Warning',
-                summary: 'The previous step failed but the workflow continued.'
+                summary: 'Failed black formatting check.'
               }
             })