Merge pull request #3156 from jakkdl/codecov_stuff

Codecov improvements

.codecov.yml

@@ -0,0 +1,29 @@
+# -- repository yaml --
+
+# Explicitly wait for all jobs to finish, as wait_for_ci prematurely triggers.
+# See https://github.com/python-trio/trio/issues/2689
+codecov:
+  notify:
+    # This number needs to be changed whenever the number of runs in CI is changed.
+    # Another option is codecov-cli: https://github.com/codecov/codecov-cli#send-notifications
+    after_n_builds: 27
+    wait_for_ci: false
+    notify_error: true # if uploads fail, replace cov comment with a comment with errors.
+  require_ci_to_pass: false
+
+  # Publicly exposing the token has some small risks from mistakes or malicious actors.
+  # See https://docs.codecov.com/docs/codecov-tokens for correctly configuring it.
+  token: 87cefb17-c44b-4f2f-8b30-1fff5769ce46
+
+# only post PR comment if coverage changes
+comment:
+  require_changes: true
+
+coverage:
+  # required range
+  range: 99.6..100
+  status:
+    # require patches to be 100%
+    patch:
+      default:
+        target: 100%

.github/workflows/ci.yml

@@ -85,9 +85,12 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           directory: empty
-          token: 87cefb17-c44b-4f2f-8b30-1fff5769ce46
           name: Windows (${{ matrix.python }}, ${{ matrix.arch }}${{ matrix.extra_name }})
+          # multiple flags is marked as an error in codecov UI, but is actually fine
+          # https://github.com/codecov/feedback/issues/567
           flags: Windows,${{ matrix.python }}
+          # this option cannot be set in .codecov.yml
+          fail_ci_if_error: true
 
   Ubuntu:
     name: 'Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})'
@@ -131,13 +134,15 @@ jobs:
         env:
           CHECK_FORMATTING: '${{ matrix.check_formatting }}'
           NO_TEST_REQUIREMENTS: '${{ matrix.no_test_requirements }}'
-      - if: always()
+      - if: >-
+          always()
+          && matrix.check_formatting != '1'
         uses: codecov/codecov-action@v3
         with:
           directory: empty
-          token: 87cefb17-c44b-4f2f-8b30-1fff5769ce46
           name: Ubuntu (${{ matrix.python }}${{ matrix.extra_name }})
           flags: Ubuntu,${{ matrix.python }}
+          fail_ci_if_error: true
 
   macOS:
     name: 'macOS (${{ matrix.python }})'
@@ -171,9 +176,9 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           directory: empty
-          token: 87cefb17-c44b-4f2f-8b30-1fff5769ce46
           name: macOS (${{ matrix.python }})
           flags: macOS,${{ matrix.python }}
+          fail_ci_if_error: true
 
   # run CI on a musl linux
   Alpine:
@@ -193,13 +198,16 @@ jobs:
         run: python -m venv .venv
       - name: Run tests
         run: source .venv/bin/activate && ./ci.sh
+      - name: Get Python version for codecov flag
+        id: get-version
+        run: echo "version=$(python -V | cut -d' ' -f2 | cut -d'.' -f1,2)" >> "${GITHUB_OUTPUT}"
       - if: always()
         uses: codecov/codecov-action@v3
         with:
           directory: empty
-          token: 87cefb17-c44b-4f2f-8b30-1fff5769ce46
           name: Alpine
-          flags: Alpine,3.12
+          flags: Alpine,${{ steps.get-version.outputs.version }}
+          fail_ci_if_error: true
 
   Cython:
     name: "Cython"