From 4f17ac7c60a0e9c29c5f18b6c1cf65feeb41246c Mon Sep 17 00:00:00 2001
From: Giovanni M Guidini <99758426+giovanni-guidini@users.noreply.github.com>
Date: Wed, 3 Jan 2024 13:19:30 -0300
Subject: [PATCH] change output of action (#32)

Currently the action sets envvars as outputs to run tests.
This is easier for the customer, but we've seen more than a few times
users hitting Linux or GHA command size limits because of the amount
of tests to run.

To solve this problem, the present changes move the output to files
created by the action.
It requires a different - more complicated - command to run tests,
as indicated in `README.md`, but should still be doable.
And should solve the command size problem too.

* use .txt versions of output that are easier to use

JSON requires more involvement of customers. .txt is Easier to use,
as suggested by @thomasrockhu-codecov

* fix some typos
* add random test in "ats_tests_to_run" if no test is selected to run
---
 .github/workflows/ci.yml | 22 ++++++++++++++--
 .gitignore               |  3 +++
 README.md                | 17 +++++++++---
 action.yml               |  5 ++--
 dist/codecov_ats.sh      | 56 ++++++++++++++++++++++-----------------
 src/codecov_ats.sh       | 57 +++++++++++++++++++++++-----------------
 6 files changed, 104 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 202c9ff..225e41c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,5 +1,13 @@
 name: Workflow for Codecov Action
 on: [push, pull_request]
+
+defaults:
+  run:
+    # the default is:
+    #      bash --noprofile --norc -eo pipefail {0}
+    # Helps with debugging
+    shell: bash --noprofile --norc -eo pipefail -ux {0}
+
 jobs:
   run:
     runs-on: ${{ matrix.os }}
@@ -19,6 +27,7 @@ jobs:
       - name: Install python dependencies
         run: pip install -r app/requirements.txt
       - name: Run ATS
+        id: run_ats
         uses: ./
         with:
           folders_to_exclude: node_modules
@@ -26,8 +35,17 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
           CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
-      - name: Run Tests and collect coverage
-        run: pytest --cov app ${{ env.CODECOV_ATS_TESTS }}
+      - name: "[debug] see content of generated files"
+        run: |
+          ls codecov_ats
+          cat codecov_ats/tests_to_run.txt
+          cat codecov_ats/tests_to_skip.txt
+      - name: Run Tests and collect coverage (if there are tests to run)
+        run: |
+            cat codecov_ats/tests_to_run.txt | xargs pytest --cov app
+      - name: "[debug] Running skipped tests"
+        run: |
+          cat codecov_ats/tests_to_skip.txt | xargs pytest --cov app
       - name: Upload to Codecov
         uses: codecov/codecov-action@v4-beta
         env:
diff --git a/.gitignore b/.gitignore
index 8e07f4d..789b67f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,6 @@ node_modules
 coverage
 __pycache__
 codecov_ats_local.sh
+
+# act
+.secrets
diff --git a/README.md b/README.md
index bb31389..abcd763 100644
--- a/README.md
+++ b/README.md
@@ -46,6 +46,7 @@ cli:
 ```
 
 5. Add the Codecov ATS Action to your CI. This should happen after you install python dependencies, but before you run tests.
+This action will populate files in `codecov_ats` folder with the tests to run.
 
 ```yaml
 - name: Run ATS
@@ -58,14 +59,17 @@ cli:
 #   run: pytest ...
 ```
 
-6. Update your `pytest` run to include the tests selected from ATS. You will need to add the `CODECOV_ATS_TESTS` variable like below.
+6. Update your `pytest` run to include the tests selected from ATS. You will read the list of tests that were selected to run by ATS.
+These tests are exported to `codecov_ats/tests_to_run.txt`. Skips running tests if no tests were selected.
+(You can copy the same step and use the `codecov_ats/tests_to_skip.txt` file to run the tests selected to be skipped)
 
 ```yaml
 - name: Run tests and collect coverage
-  run: pytest --cov app ${{ env.CODECOV_ATS_TESTS }}
+  run: |
+    cat codecov_ats/tests_to_run.txt | xargs pytest --cov app
 ```
 
-7. If you are not already using the Codecov CLI to upload coverage, you can update the Codecov Action to `v4-beta`
+1. If you are not already using the Codecov CLI to upload coverage, you can update the Codecov Action to `v4-beta`
 
 ```yaml
 - name: Upload coverage to Codecov
@@ -78,3 +82,10 @@ cli:
 ```
 
 8. Run your CI! On your first run, Codecov will not have any labels data and will have to run all tests. However, once all following commits or pull requests are rebased on top of this commit, you should be able to see the benefits of ATS.
+
+### Output
+
+This action creates a `codecov_ats` folder in the current directory and populates it with 3 files:
+1. `codecov_ats/tests_to_run.json` - List of tests selected by Automated Test Selection that should be executed
+2. `codecov_ats/tests_to_skip.json` - List of tests that are being skiped
+3. `codecov_ats/result.json` - Summary of results for test selection
\ No newline at end of file
diff --git a/action.yml b/action.yml
index dfe6780..e6de88e 100644
--- a/action.yml
+++ b/action.yml
@@ -84,9 +84,8 @@ runs:
         ${{ github.action_path }}/dist/codecov_ats.sh | tee codecov_ats_output.txt
 
         if [[ $? == 0 ]]; then
-          echo "Setting CODECOV_ATS_TESTS to GitHub environment"
-          commands=$(tail -1 codecov_ats_output.txt)
-          echo "CODECOV_ATS_TESTS=$commands" >> "$GITHUB_ENV"
+          echo "Codecov: Action complete. Check codecov_ats folder for results."
+          cat codecov_ats/result.json
         else
           echo "Codecov: Action failed to successfully run"
           exit 1;
diff --git a/dist/codecov_ats.sh b/dist/codecov_ats.sh
index 6473800..f17fcd4 100755
--- a/dist/codecov_ats.sh
+++ b/dist/codecov_ats.sh
@@ -115,31 +115,39 @@ if [[ -z $response && -n $INPUTS_OVERRIDE_BASE_COMMIT ]]; then
 	exit 1
 fi
 
-response=$(echo $response | sed 's/,//g')
-runner_options=$(echo $response | sed 's/^.*runner_options\": \[//' | sed 's/\].*$//')
-ats_tests_to_run=$(echo $response | sed 's/^.*ats_tests_to_run\": \[//' | sed 's/\].*$//')
-ats_tests_to_skip=$(echo $response | sed 's/^.*ats_tests_to_skip\": \[//' | sed 's/\].*$//')
-
-if [[ -z $runner_options ]]; then
-    say "$y==>$x Could not find 'runner_options', defaulting to '--cov-context=test'"
-    runner_options="--cov-context=test"
+# Post process label-analysis response
+
+# Create directory to put result files
+mkdir codecov_ats
+# Export tests to run and tests to skip into respective files
+jq <<< "$response" '.runner_options + .ats_tests_to_run | @sh' --raw-output > codecov_ats/tests_to_run.txt
+jq <<< "$response" '.runner_options + .ats_tests_to_skip | @sh' --raw-output > codecov_ats/tests_to_skip.txt
+
+
+# Statistics on the test selection
+testcount() { jq <<< "$response" ".$1 | length"; }
+run_count=$(testcount ats_tests_to_run)
+skip_count=$(testcount ats_tests_to_skip)
+
+# Change tests_to_run to have 1 test if no tests were selected
+# This avoids users running ALL tests if no test is selected to run
+# ⚠️ it's safer for the customer if they check test counts themselves and run tests conditionally
+if [[ "$run_count" -eq 0 ]]; then
+    say "All tests skipped. Adding random test in tests_to_run to avoid running all tests"
+    jq <<< "$response" --argjson randint $RANDOM '.runner_options + [.ats_tests_to_skip[$randint % length]] | @sh' --raw-output > codecov_ats/tests_to_run.txt
 fi
 
-if [[ -z $ats_tests_to_run ]]; then
-    say "$y==>$x No tests to run, picking random test"
-    ats_tests_to_skip_array=($ats_tests_to_skip)
-
-    if [[ -z $ats_tests_to_skip ]]; then
-        say "$y==>$x No tests to skip, running all tests"
-    else
-        ats_tests_to_run=${ats_tests_to_skip_array[ $RANDOM % ${#ats_tests_to_skip_array[@]} ]}
-    fi
-elif [[ -z $ats_tests_to_skip ]]; then
-    say "$y==>$x No tests are skipped, running all"
-    ats_tests_to_run=""
+# Parse any potential errors that made ATS fallback to running all tests and surface them
+ats_fallback_reason=$(jq <<< "$response" '.ats_fallback_reason')
+if [ "$ats_fallback_reason" == "null" ]; then
+    ats_success=true
+else
+    ats_success=false
 fi
+tee <<< \
+    "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
+    "$GITHUB_STEP_SUMMARY" \
+    "codecov_ats/result.json"
 
-test_commands="$runner_options "
-test_commands+=$ats_tests_to_run
-say "${g}Arguments to run:$x"
-echo "$test_commands"
+echo "Tests to run exported to ./codecov_ats/tests_to_run.txt"
+echo "Tests to skip exported to ./codecov_ats/tests_to_skip.txt"
diff --git a/src/codecov_ats.sh b/src/codecov_ats.sh
index 6473800..fd0dc0e 100755
--- a/src/codecov_ats.sh
+++ b/src/codecov_ats.sh
@@ -115,31 +115,40 @@ if [[ -z $response && -n $INPUTS_OVERRIDE_BASE_COMMIT ]]; then
 	exit 1
 fi
 
-response=$(echo $response | sed 's/,//g')
-runner_options=$(echo $response | sed 's/^.*runner_options\": \[//' | sed 's/\].*$//')
-ats_tests_to_run=$(echo $response | sed 's/^.*ats_tests_to_run\": \[//' | sed 's/\].*$//')
-ats_tests_to_skip=$(echo $response | sed 's/^.*ats_tests_to_skip\": \[//' | sed 's/\].*$//')
-
-if [[ -z $runner_options ]]; then
-    say "$y==>$x Could not find 'runner_options', defaulting to '--cov-context=test'"
-    runner_options="--cov-context=test"
-fi
+# Post process label-analysis response
 
-if [[ -z $ats_tests_to_run ]]; then
-    say "$y==>$x No tests to run, picking random test"
-    ats_tests_to_skip_array=($ats_tests_to_skip)
 
-    if [[ -z $ats_tests_to_skip ]]; then
-        say "$y==>$x No tests to skip, running all tests"
-    else
-        ats_tests_to_run=${ats_tests_to_skip_array[ $RANDOM % ${#ats_tests_to_skip_array[@]} ]}
-    fi
-elif [[ -z $ats_tests_to_skip ]]; then
-    say "$y==>$x No tests are skipped, running all"
-    ats_tests_to_run=""
+# Create directory to put result files
+mkdir codecov_ats
+# Export tests to run and tests to skip into respective files
+jq <<< "$response" '.runner_options + .ats_tests_to_run | @sh' --raw-output > codecov_ats/tests_to_run.txt
+jq <<< "$response" '.runner_options + .ats_tests_to_skip | @sh' --raw-output > codecov_ats/tests_to_skip.txt
+
+
+# Statistics on the test selection
+testcount() { jq <<< "$response" ".$1 | length"; }
+run_count=$(testcount ats_tests_to_run)
+skip_count=$(testcount ats_tests_to_skip)
+
+# Change tests_to_run to have 1 test if no tests were selected
+# This avoids users running ALL tests if no test is selected to run
+# (ideally customer will they check test counts themselves and run tests conditionally)
+if [[ "$run_count" -eq 0 ]]; then
+    say "All tests skipped. Adding random test in tests_to_run to avoid running all tests"
+    jq <<< "$response" --argjson randint $RANDOM '.runner_options + [.ats_tests_to_skip[$randint % length]] | @sh' --raw-output > codecov_ats/tests_to_run.txt
+fi
+
+# Parse any potential errors that made ATS fallback to running all tests and surface them
+ats_fallback_reason=$(jq <<< "$response" '.ats_fallback_reason')
+if [ "$ats_fallback_reason" == "null" ]; then
+    ats_success=true
+else
+    ats_success=false
 fi
+tee <<< \
+    "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
+    "$GITHUB_STEP_SUMMARY" \
+    "codecov_ats/result.json"
 
-test_commands="$runner_options "
-test_commands+=$ats_tests_to_run
-say "${g}Arguments to run:$x"
-echo "$test_commands"
+echo "Tests to run exported to ./codecov_ats/tests_to_run.txt"
+echo "Tests to skip exported to ./codecov_ats/tests_to_skip.txt"