change output of action (#32)

Currently the action sets envvars as outputs to run tests. This is easier for the customer, but we've seen more than a few times users hitting Linux or GHA command size limits because of the amount of tests to run. To solve this problem, the present changes move the output to files created by the action. It requires a different - more complicated - command to run tests, as indicated in `README.md`, but should still be doable. And should solve the command size problem too. * use .txt versions of output that are easier to use JSON requires more involvement of customers. .txt is Easier to use, as suggested by @thomasrockhu-codecov * fix some typos * add random test in "ats_tests_to_run" if no test is selected to run
codecov · Jan 3, 2024 · 4f17ac7 · 4f17ac7
1 parent ccd3eab
commit 4f17ac7
Show file tree

Hide file tree

Showing 6 changed files with 104 additions and 56 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,5 +1,13 @@
 name: Workflow for Codecov Action
 on: [push, pull_request]
+
+defaults:
+  run:
+    # the default is:
+    #      bash --noprofile --norc -eo pipefail {0}
+    # Helps with debugging
+    shell: bash --noprofile --norc -eo pipefail -ux {0}
+
 jobs:
   run:
     runs-on: ${{ matrix.os }}
@@ -19,15 +27,25 @@ jobs:
       - name: Install python dependencies
         run: pip install -r app/requirements.txt
       - name: Run ATS
+        id: run_ats
         uses: ./
         with:
           folders_to_exclude: node_modules
           verbose: true
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
           CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
-      - name: Run Tests and collect coverage
-        run: pytest --cov app ${{ env.CODECOV_ATS_TESTS }}
+      - name: "[debug] see content of generated files"
+        run: |
+          ls codecov_ats
+          cat codecov_ats/tests_to_run.txt
+          cat codecov_ats/tests_to_skip.txt
+      - name: Run Tests and collect coverage (if there are tests to run)
+        run: |
+            cat codecov_ats/tests_to_run.txt | xargs pytest --cov app
+      - name: "[debug] Running skipped tests"
+        run: |
+          cat codecov_ats/tests_to_skip.txt | xargs pytest --cov app
       - name: Upload to Codecov
         uses: codecov/codecov-action@v4-beta
         env:

diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,6 @@ node_modules
 coverage
 __pycache__
 codecov_ats_local.sh
+
+# act
+.secrets
diff --git a/README.md b/README.md
@@ -46,6 +46,7 @@ cli:
 ```
 
 5. Add the Codecov ATS Action to your CI. This should happen after you install python dependencies, but before you run tests.
+This action will populate files in `codecov_ats` folder with the tests to run.
 
 ```yaml
 - name: Run ATS
@@ -58,14 +59,17 @@ cli:
 #   run: pytest ...
 ```
 
-6. Update your `pytest` run to include the tests selected from ATS. You will need to add the `CODECOV_ATS_TESTS` variable like below.
+6. Update your `pytest` run to include the tests selected from ATS. You will read the list of tests that were selected to run by ATS.
+These tests are exported to `codecov_ats/tests_to_run.txt`. Skips running tests if no tests were selected.
+(You can copy the same step and use the `codecov_ats/tests_to_skip.txt` file to run the tests selected to be skipped)
 
 ```yaml
 - name: Run tests and collect coverage
-  run: pytest --cov app ${{ env.CODECOV_ATS_TESTS }}
+  run: |
+    cat codecov_ats/tests_to_run.txt | xargs pytest --cov app
 ```
 
-7. If you are not already using the Codecov CLI to upload coverage, you can update the Codecov Action to `v4-beta`
+1. If you are not already using the Codecov CLI to upload coverage, you can update the Codecov Action to `v4-beta`
 
 ```yaml
 - name: Upload coverage to Codecov
@@ -78,3 +82,10 @@ cli:
 ```
 
 8. Run your CI! On your first run, Codecov will not have any labels data and will have to run all tests. However, once all following commits or pull requests are rebased on top of this commit, you should be able to see the benefits of ATS.
+
+### Output
+
+This action creates a `codecov_ats` folder in the current directory and populates it with 3 files:
+1. `codecov_ats/tests_to_run.json` - List of tests selected by Automated Test Selection that should be executed
+2. `codecov_ats/tests_to_skip.json` - List of tests that are being skiped
+3. `codecov_ats/result.json` - Summary of results for test selection
diff --git a/action.yml b/action.yml
@@ -84,9 +84,8 @@ runs:
         ${{ github.action_path }}/dist/codecov_ats.sh | tee codecov_ats_output.txt
 
         if [[ $? == 0 ]]; then
-          echo "Setting CODECOV_ATS_TESTS to GitHub environment"
-          commands=$(tail -1 codecov_ats_output.txt)
-          echo "CODECOV_ATS_TESTS=$commands" >> "$GITHUB_ENV"
+          echo "Codecov: Action complete. Check codecov_ats folder for results."
+          cat codecov_ats/result.json
         else
           echo "Codecov: Action failed to successfully run"
           exit 1;

diff --git a/dist/codecov_ats.sh b/dist/codecov_ats.sh
@@ -115,31 +115,39 @@ if [[ -z $response && -n $INPUTS_OVERRIDE_BASE_COMMIT ]]; then
 	exit 1
 fi
 
-response=$(echo $response | sed 's/,//g')
-runner_options=$(echo $response | sed 's/^.*runner_options\": \[//' | sed 's/\].*$//')
-ats_tests_to_run=$(echo $response | sed 's/^.*ats_tests_to_run\": \[//' | sed 's/\].*$//')
-ats_tests_to_skip=$(echo $response | sed 's/^.*ats_tests_to_skip\": \[//' | sed 's/\].*$//')
-
-if [[ -z $runner_options ]]; then
-    say "$y==>$x Could not find 'runner_options', defaulting to '--cov-context=test'"
-    runner_options="--cov-context=test"
+# Post process label-analysis response
+
+# Create directory to put result files
+mkdir codecov_ats
+# Export tests to run and tests to skip into respective files
+jq <<< "$response" '.runner_options + .ats_tests_to_run | @sh' --raw-output > codecov_ats/tests_to_run.txt
+jq <<< "$response" '.runner_options + .ats_tests_to_skip | @sh' --raw-output > codecov_ats/tests_to_skip.txt
+
+
+# Statistics on the test selection
+testcount() { jq <<< "$response" ".$1 | length"; }
+run_count=$(testcount ats_tests_to_run)
+skip_count=$(testcount ats_tests_to_skip)
+
+# Change tests_to_run to have 1 test if no tests were selected
+# This avoids users running ALL tests if no test is selected to run
+# ⚠️ it's safer for the customer if they check test counts themselves and run tests conditionally
+if [[ "$run_count" -eq 0 ]]; then
+    say "All tests skipped. Adding random test in tests_to_run to avoid running all tests"
+    jq <<< "$response" --argjson randint $RANDOM '.runner_options + [.ats_tests_to_skip[$randint % length]] | @sh' --raw-output > codecov_ats/tests_to_run.txt
 fi
 
-if [[ -z $ats_tests_to_run ]]; then
-    say "$y==>$x No tests to run, picking random test"
-    ats_tests_to_skip_array=($ats_tests_to_skip)
-
-    if [[ -z $ats_tests_to_skip ]]; then
-        say "$y==>$x No tests to skip, running all tests"
-    else
-        ats_tests_to_run=${ats_tests_to_skip_array[ $RANDOM % ${#ats_tests_to_skip_array[@]} ]}
-    fi
-elif [[ -z $ats_tests_to_skip ]]; then
-    say "$y==>$x No tests are skipped, running all"
-    ats_tests_to_run=""
+# Parse any potential errors that made ATS fallback to running all tests and surface them
+ats_fallback_reason=$(jq <<< "$response" '.ats_fallback_reason')
+if [ "$ats_fallback_reason" == "null" ]; then
+    ats_success=true
+else
+    ats_success=false
 fi
+tee <<< \
+    "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
+    "$GITHUB_STEP_SUMMARY" \
+    "codecov_ats/result.json"
 
-test_commands="$runner_options "
-test_commands+=$ats_tests_to_run
-say "${g}Arguments to run:$x"
-echo "$test_commands"
+echo "Tests to run exported to ./codecov_ats/tests_to_run.txt"
+echo "Tests to skip exported to ./codecov_ats/tests_to_skip.txt"
diff --git a/src/codecov_ats.sh b/src/codecov_ats.sh
@@ -115,31 +115,40 @@ if [[ -z $response && -n $INPUTS_OVERRIDE_BASE_COMMIT ]]; then
 	exit 1
 fi
 
-response=$(echo $response | sed 's/,//g')
-runner_options=$(echo $response | sed 's/^.*runner_options\": \[//' | sed 's/\].*$//')
-ats_tests_to_run=$(echo $response | sed 's/^.*ats_tests_to_run\": \[//' | sed 's/\].*$//')
-ats_tests_to_skip=$(echo $response | sed 's/^.*ats_tests_to_skip\": \[//' | sed 's/\].*$//')
-
-if [[ -z $runner_options ]]; then
-    say "$y==>$x Could not find 'runner_options', defaulting to '--cov-context=test'"
-    runner_options="--cov-context=test"
-fi
+# Post process label-analysis response
 
-if [[ -z $ats_tests_to_run ]]; then
-    say "$y==>$x No tests to run, picking random test"
-    ats_tests_to_skip_array=($ats_tests_to_skip)
 
-    if [[ -z $ats_tests_to_skip ]]; then
-        say "$y==>$x No tests to skip, running all tests"
-    else
-        ats_tests_to_run=${ats_tests_to_skip_array[ $RANDOM % ${#ats_tests_to_skip_array[@]} ]}
-    fi
-elif [[ -z $ats_tests_to_skip ]]; then
-    say "$y==>$x No tests are skipped, running all"
-    ats_tests_to_run=""
+# Create directory to put result files
+mkdir codecov_ats
+# Export tests to run and tests to skip into respective files
+jq <<< "$response" '.runner_options + .ats_tests_to_run | @sh' --raw-output > codecov_ats/tests_to_run.txt
+jq <<< "$response" '.runner_options + .ats_tests_to_skip | @sh' --raw-output > codecov_ats/tests_to_skip.txt
+
+
+# Statistics on the test selection
+testcount() { jq <<< "$response" ".$1 | length"; }
+run_count=$(testcount ats_tests_to_run)
+skip_count=$(testcount ats_tests_to_skip)
+
+# Change tests_to_run to have 1 test if no tests were selected
+# This avoids users running ALL tests if no test is selected to run
+# (ideally customer will they check test counts themselves and run tests conditionally)
+if [[ "$run_count" -eq 0 ]]; then
+    say "All tests skipped. Adding random test in tests_to_run to avoid running all tests"
+    jq <<< "$response" --argjson randint $RANDOM '.runner_options + [.ats_tests_to_skip[$randint % length]] | @sh' --raw-output > codecov_ats/tests_to_run.txt
+fi
+
+# Parse any potential errors that made ATS fallback to running all tests and surface them
+ats_fallback_reason=$(jq <<< "$response" '.ats_fallback_reason')
+if [ "$ats_fallback_reason" == "null" ]; then
+    ats_success=true
+else
+    ats_success=false
 fi
+tee <<< \
+    "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
+    "$GITHUB_STEP_SUMMARY" \
+    "codecov_ats/result.json"
 
-test_commands="$runner_options "
-test_commands+=$ats_tests_to_run
-say "${g}Arguments to run:$x"
-echo "$test_commands"
+echo "Tests to run exported to ./codecov_ats/tests_to_run.txt"
+echo "Tests to skip exported to ./codecov_ats/tests_to_skip.txt"