Skip to content

Commit

Permalink
chore: backport advanced test updates (#2140)
Browse files Browse the repository at this point in the history 
* test: run advanced tests nightly to detect breakages (#2064)

* test: run advanced tests nightly to detect breakages

* update schedule

* test: ensure advanced tests fail if orchestrator fails (#2068)

* test: ensure advanced tests fail if orchestrator fails

* separate steps

---------

Co-authored-by: Lucas Bertrand <[email protected]>

* ci: add slack notifications nightly/develop failures (#2078)

* ci: add slack notifications nightly/develop failures

* event_name

---------

Co-authored-by: Lucas Bertrand <[email protected]>
Co-authored-by: Grant Zukel <[email protected]>
  • Loading branch information
@gartnera @lumtis @gzukel
3 people authored May 8, 2024
1 parent 76439ba commit b05c2fa
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 114 deletions.
9 changes: 9 additions & 0 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,15 @@ jobs:
exit 1
fi
- name: Notify Slack on Failure
if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
uses: 8398a7/action-slack@v3
with:
status: ${{ job.status }}
fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}

- name: Stop Private Network
if: always()
run: |
Expand Down
163 changes: 49 additions & 114 deletions .github/workflows/execute_advanced_tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,85 +23,64 @@ on:
type: boolean
required: false
default: false
schedule:
# run at 6AM UTC Daily
# 6AM UTC -> 11PM PT
- cron: "0 6 * * *"

jobs:
e2e-admin-tests:
if: ${{ github.event.inputs.e2e-admin-tests == 'true' }}
if: ${{ github.event.inputs.e2e-admin-tests == 'true' || github.event_name == 'schedule' }}
runs-on: buildjet-4vcpu-ubuntu-2204
timeout-minutes: 120
steps:
- name: "Checkout Code"
uses: actions/checkout@v3

- name: Execute e2e-admin-tests
shell: bash
- name: Start Test
run: make start-e2e-admin-test

# use docker logs -f rather than docker attach to make sure we get the initial logs
- name: Watch Test
run: |
make start-e2e-admin-test
container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
if [ -z "$container_id" ]; then
echo "Orchestrator container is not currently running. Exiting..."
exit 1
fi
echo "Monitoring Orchestrator container with ID: $container_id"
while true; do
current_status=$(docker ps -q | grep $container_id || echo "docker-ps-error")
if [ "$current_status" == "docker-ps-error" ]; then
echo "***********************************"
echo "* *"
echo "* *"
echo "* TESTING COMPLETE *"
echo "* *"
echo "* *"
echo "***********************************"
docker logs $container_id || echo "no logs"
exit 0
else
echo "Testing in progress still...."
if [ "${{ github.event.inputs.debug }}" == "true" ]; then
docker logs $container_id || echo "no logs"
fi
fi
sleep 5
done
docker logs -f "${container_id}" &
exit $(docker wait "${container_id}")
- name: Notify Slack on Failure
if: failure() && github.event_name == 'schedule'
uses: 8398a7/action-slack@v3
with:
status: ${{ job.status }}
fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}

e2e-upgrade-test:
if: ${{ github.event.inputs.e2e-upgrade-test == 'true' }}
if: ${{ github.event.inputs.e2e-upgrade-test == 'true' || github.event_name == 'schedule' }}
runs-on: buildjet-4vcpu-ubuntu-2204
timeout-minutes: 120
steps:
- name: "Checkout Code"
uses: actions/checkout@v3

- name: Execute upgrade-test
shell: bash
- name: Start Test
run: make start-upgrade-test

- name: Watch Test
run: |
make start-upgrade-test
container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
if [ -z "$container_id" ]; then
echo "Orchestrator container is not currently running. Exiting..."
exit 1
fi
echo "Monitoring Orchestrator container with ID: $container_id"
while true; do
current_status=$(docker ps -q | grep $container_id || echo "docker-ps-error")
if [ "$current_status" == "docker-ps-error" ]; then
echo "***********************************"
echo "* *"
echo "* *"
echo "* TESTING COMPLETE *"
echo "* *"
echo "* *"
echo "***********************************"
docker logs $container_id || echo "no logs"
exit 0
else
echo "Testing in progress still...."
if [ "${{ github.event.inputs.debug }}" == "true" ]; then
docker logs $container_id || echo "no logs"
fi
fi
sleep 5
done
docker logs -f "${container_id}" &
exit $(docker wait "${container_id}")
- name: Notify Slack on Failure
if: failure() && github.event_name == 'schedule'
uses: 8398a7/action-slack@v3
with:
status: ${{ job.status }}
fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}

e2e-upgrade-test-light:
if: ${{ github.event.inputs.e2e-upgrade-test-light == 'true' }}
Expand All @@ -111,36 +90,14 @@ jobs:
- name: "Checkout Code"
uses: actions/checkout@v3

- name: Execute upgrade-test-light
shell: bash
- name: Start Test
run: make start-upgrade-test-light

- name: Watch Test
run: |
make start-upgrade-test-light
container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
if [ -z "$container_id" ]; then
echo "Orchestrator container is not currently running. Exiting..."
exit 1
fi
echo "Monitoring Orchestrator container with ID: $container_id"
while true; do
current_status=$(docker ps -q | grep $container_id || echo "docker-ps-error")
if [ "$current_status" == "docker-ps-error" ]; then
echo "***********************************"
echo "* *"
echo "* *"
echo "* TESTING COMPLETE *"
echo "* *"
echo "* *"
echo "***********************************"
docker logs $container_id || echo "no logs"
exit 0
else
echo "Testing in progress still...."
if [ "${{ github.event.inputs.debug }}" == "true" ]; then
docker logs $container_id || echo "no logs"
fi
fi
sleep 5
done
docker logs -f "${container_id}" &
exit $(docker wait "${container_id}")
e2e-performance-test:
if: ${{ github.event.inputs.e2e-performance-test == 'true' }}
Expand All @@ -150,33 +107,11 @@ jobs:
- name: "Checkout Code"
uses: actions/checkout@v3

- name: Execute Performance Tests
shell: bash
- name: Start Test
run: make start-e2e-performance-test

- name: Watch Test
run: |
make start-e2e-performance-test
container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
if [ -z "$container_id" ]; then
echo "Orchestrator container is not currently running. Exiting..."
exit 1
fi
echo "Monitoring Orchestrator container with ID: $container_id"
while true; do
current_status=$(docker ps -q | grep $container_id || echo "docker-ps-error")
if [ "$current_status" == "docker-ps-error" ]; then
echo "***********************************"
echo "* *"
echo "* *"
echo "* TESTING COMPLETE *"
echo "* *"
echo "* *"
echo "***********************************"
docker logs $container_id || echo "no logs"
exit 0
else
echo "Testing in progress still...."
if [ "${{ github.event.inputs.debug }}" == "true" ]; then
docker logs $container_id || echo "no logs"
fi
fi
sleep 5
done
docker logs -f "${container_id}" &
exit $(docker wait "${container_id}")

0 comments on commit b05c2fa

Please sign in to comment.