Update Notification Condition for Scheduled Test Runs (#8115)

# Description

Updating scheduled test run notification logic. With this update every
scheduled functional/long running test run failure will log a new issue.

## Type of change

<!--

Please select **one** of the following options that describes your
change and delete the others. Clearly identifying the type of change you
are making will help us review your PR faster, and is used in authoring
release notes.

If you are making a bug fix or functionality change to Radius and do not
have an associated issue link please create one now.

-->

- This pull request fixes a bug in Radius and has an approved issue
(issue link required).

<!--

Please update the following to link the associated issue. This is
required for some kinds of changes (see above).

-->

Fixes: #7998

## Contributor checklist
Please verify that the PR meets the following requirements, where
applicable:

- [ ] An overview of proposed schema changes is included in a linked
GitHub issue. -- N/A
- [ ] A design document PR is created in the [design-notes
repository](https://github.com/radius-project/design-notes/), if new
APIs are being introduced. -- N/A
- [ ] If applicable, design document has been reviewed and approved by
Radius maintainers/approvers. -- N/A
- [ ] A PR for the [samples
repository](https://github.com/radius-project/samples) is created, if
existing samples are affected by the changes in this PR. -- N/A
- [ ] A PR for the [documentation
repository](https://github.com/radius-project/docs) is created, if the
changes in this PR affect the documentation or any user facing updates
are made. -- N/A
- [ ] A PR for the [recipes
repository](https://github.com/radius-project/recipes) is created, if
existing recipes are affected by the changes in this PR. -- N/A

Signed-off-by: karishma-chawla <[email protected]>
Co-authored-by: karishma-chawla <[email protected]>

.github/workflows/functional-test-cloud.yaml

@@ -85,8 +85,6 @@ env:
   FUNCTIONAL_TEST_APP_ID: 425843
   # Private Git repository where terraform module for testing is stored.
   TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"
-  # The number of failed tests to report.
-  ISSUE_CREATE_THRESHOLD: 2
   # bicep-types ACR url for uploading Radius Bicep types
   BICEP_TYPES_REGISTRY: 'biceptypes.azurecr.io'
 
@@ -894,39 +892,14 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius'
     steps:
-      - name: Count recently failed tests
-        id: count_failures
-        uses: actions/github-script@v7
-        with:
-          script: |
-            response = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'functional-test-cloud.yaml',
-              event: 'schedule',
-              per_page: 10
-            });
-
-            failureCount = 1;
-            for (const run of response.data.workflow_runs) {
-              if (run.conclusion === 'failure') {
-                failureCount++;
-              } else {
-                break;
-              }
-            }
-            return failureCount;
-
       - name: Create failure issue for failing scheduled run
         uses: actions/github-script@v7
-        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
-        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled functional test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
-            })
+              body: `## Bug information \n\nThis issue is automatically generated if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+            })

.github/workflows/functional-test-noncloud.yaml

@@ -66,10 +66,6 @@ env:
   TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local"
   # Private Git repository where terraform module for testing is stored.
   TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"
-
-  # The number of failed tests to report.
-  ISSUE_CREATE_THRESHOLD: 2
-
   # Local Docker registry name
   LOCAL_REGISTRY_NAME: "radius-registry"
   # Local Docker registry server
@@ -453,38 +449,14 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius'
     steps:
-      - name: Count recently failed tests
-        id: count_failures
-        uses: actions/github-script@v7
-        with:
-          script: |
-            response = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'functional-test-noncloud.yaml',
-              event: 'schedule',
-              per_page: 10
-            });
-
-            failureCount = 1;
-            for (const run of response.data.workflow_runs) {
-              if (run.conclusion === 'failure') {
-                failureCount++;
-              } else {
-                break;
-              }
-            }
-            return failureCount;
       - name: Create failure issue for failing scheduled run
         uses: actions/github-script@v7
-        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
-        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled functional test (noncloud) failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis issue is automatically generated if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })

.github/workflows/long-running-azure.yaml

@@ -95,9 +95,6 @@ env:
   # The current GitHub action link
   ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
-  # The number of failed tests to report.
-  ISSUE_CREATE_THRESHOLD: 2
-
 jobs:
   build:
     name: Build Radius for test
@@ -522,38 +519,14 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.repository == 'radius-project/radius' && github.event_name == 'schedule'
     steps:
-      - name: Count recently failed tests
-        id: count_failures
-        uses: actions/github-script@v7
-        with:
-          script: |
-            response = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'long-running-azure.yaml',
-              event: 'schedule',
-              per_page: 10
-            });
-
-            failureCount = 1;
-            for (const run of response.data.workflow_runs) {
-              if (run.conclusion === 'failure') {
-                failureCount++;
-              } else {
-                break;
-              }
-            }
-            return failureCount;
       - name: Create failure issue for failing long running test run
         uses: actions/github-script@v7
-        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
-        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled long running test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled long running test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis issue is automatically generated if the scheduled long running test fails. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })